DGNum/Stirling-PDF
15
0
Fork 0
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity

pipeline refactor beginnings

Browse source
This commit is contained in:
Anthony Stirling 2023-12-28 17:23:19 +00:00
parent 8acab77ae3
commit 5fcb4e893b
3 changed files with 273 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java
View file

@ -1,6 +1,8 @@
package stirling.software.SPDF.controller.api.pipeline;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
@ -41,6 +43,9 @@ public class ApiDocService {
return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
}
JsonNode apiDocsJsonRootNode;
//@EventListener(ApplicationReadyEvent.class)
private synchronized void loadApiDocumentation() {
try {
@ -56,9 +61,9 @@ public class ApiDocService {
String apiDocsJson = response.getBody();
ObjectMapper mapper = new ObjectMapper();
JsonNode root = mapper.readTree(apiDocsJson);
apiDocsJsonRootNode = mapper.readTree(apiDocsJson);
JsonNode paths = root.path("paths");
JsonNode paths = apiDocsJsonRootNode.path("paths");
paths.fields().forEachRemaining(entry -> {
String path = entry.getKey();
JsonNode pathNode = entry.getValue();
@ -84,6 +89,27 @@ public class ApiDocService {
ApiEndpoint endpoint = apiDocumentation.get(operationName);
return endpoint.areParametersValid(parameters);
}
public boolean isMultiInput(String operationName) {
if(apiDocsJsonRootNode == null || apiDocumentation.size() == 0) {
loadApiDocumentation();
}
if (!apiDocumentation.containsKey(operationName)) {
return false;
}
ApiEndpoint endpoint = apiDocumentation.get(operationName);
String description = endpoint.getDescription();
Pattern pattern = Pattern.compile("Type:(\\w+)");
Matcher matcher = pattern.matcher(description);
if (matcher.find()) {
String type = matcher.group(1);
return type.startsWith("MI");
}
return false;
}
}
// Model class for API Endpoint

176
src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java
View file

@ -15,6 +15,7 @@ import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -73,12 +74,9 @@ public class PipelineController {
@Autowired
private ApiDocService apiDocService;
@Scheduled(fixedRate = 60000)
public void scanFolders() {
if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
return;
}
Path watchedFolderPath = Paths.get(watchedFoldersDir);
@ -106,26 +104,6 @@ public class PipelineController {
}
}
@Autowired
ApplicationProperties applicationProperties;
@Autowired(required=false)
private UserServiceInterface userService;
private String getApiKeyForUser() {
if(userService == null)
return "";
return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
}
@Autowired
private ServletContext servletContext;
private String getBaseUrl() {
String contextPath = servletContext.getContextPath();
return "http://localhost:8080" + contextPath + "/";
}
private void handleDirectory(Path dir) throws Exception {
logger.info("Handling directory: {}", dir);
@ -139,13 +117,9 @@ public class PipelineController {
Optional<Path> jsonFileOptional;
// Find any JSON file in the directory
try (Stream<Path> paths = Files.list(dir)) {
jsonFileOptional = paths
.filter(file -> file.toString().endsWith(".json"))
.findFirst();
jsonFileOptional = paths.filter(file -> file.toString().endsWith(".json")).findFirst();
}
if (jsonFileOptional.isPresent()) {
jsonFile = jsonFileOptional.get();
// Read JSON file
@ -185,11 +159,9 @@ public class PipelineController {
if ("automated".equals(fileInput)) {
// If fileInput is "automated", process all files in the directory
try (Stream<Path> paths = Files.list(dir)) {
files = paths
.filter(path -> !Files.isDirectory(path)) // exclude directories
files = paths.filter(path -> !Files.isDirectory(path)) // exclude directories
.filter(path -> !path.equals(jsonFile)) // exclude jsonFile
.map(Path::toFile)
.toArray(File[]::new);
.map(Path::toFile).toArray(File[]::new);
} catch (IOException e) {
e.printStackTrace();
@ -204,7 +176,7 @@ public class PipelineController {
List<File> filesToProcess = new ArrayList<>();
for (File file : files) {
logger.info(file.getName());
logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName()));
logger.info("{} to {}", file.toPath(), processingDir.resolve(file.getName()));
Files.move(file.toPath(), processingDir.resolve(file.getName()));
filesToProcess.add(processingDir.resolve(file.getName()).toFile());
}
@ -213,14 +185,14 @@ public class PipelineController {
try {
List<Resource> resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString);
if(resources == null) {
if (resources == null) {
return;
}
// Move resultant files and rename them as per config in JSON file
for (Resource resource : resources) {
String resourceName = resource.getFilename();
String baseName = resourceName.substring(0, resourceName.lastIndexOf("."));
String extension = resourceName.substring(resourceName.lastIndexOf(".")+1);
String extension = resourceName.substring(resourceName.lastIndexOf(".") + 1);
String outputFileName = config.getOutputPattern().replace("{filename}", baseName);
@ -272,10 +244,11 @@ public class PipelineController {
}
}
logger.info("outputPath {}", outputPath);
logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString());
logger.info("outputPath.resolve(outputFileName).toString() {}",
outputPath.resolve(outputFileName).toString());
File newFile = new File(outputPath.resolve(outputFileName).toString());
OutputStream os = new FileOutputStream(newFile);
os.write(((ByteArrayResource)resource).getByteArray());
os.write(((ByteArrayResource) resource).getByteArray());
os.close();
logger.info("made {}", outputPath.resolve(outputFileName));
}
@ -297,6 +270,26 @@ public class PipelineController {
}
}
@Autowired
ApplicationProperties applicationProperties;
@Autowired(required = false)
private UserServiceInterface userService;
private String getApiKeyForUser() {
if (userService == null)
return "";
return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
}
@Autowired
private ServletContext servletContext;
private String getBaseUrl() {
String contextPath = servletContext.getContextPath();
return "http://localhost:8080" + contextPath + "/";
}
List<Resource> processFiles(List<Resource> outputFiles, String jsonString) throws Exception {
ObjectMapper mapper = new ObjectMapper();
@ -313,7 +306,9 @@ public class PipelineController {
for (JsonNode operationNode : pipelineNode) {
String operation = operationNode.get("operation").asText();
logger.info("Running operation: {}", operation);
boolean isMultiInputOperation = apiDocService.isMultiInput(operation);
logger.info("Running operation: {} isMultiInputOperation {}", operation, isMultiInputOperation);
JsonNode parametersNode = operationNode.get("parameters");
String inputFileExtension = "";
if (operationNode.has("inputFileType")) {
@ -321,11 +316,17 @@ public class PipelineController {
} else {
inputFileExtension = ".pdf";
}
final String finalInputFileExtension = inputFileExtension;
RestTemplate restTemplate = new RestTemplate();
String url = getBaseUrl() + operation;
List<Resource> newOutputFiles = new ArrayList<>();
boolean hasInputFileType = false;
if (!isMultiInputOperation) {
for (Resource file : outputFiles) {
boolean hasInputFileType = false;
if (file.getFilename().endsWith(inputFileExtension)) {
hasInputFileType = true;
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
@ -345,13 +346,13 @@ public class PipelineController {
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
RestTemplate restTemplate = new RestTemplate();
String url = getBaseUrl() + operation;
ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity,
byte[].class);
ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
// If the operation is filter and the response body is null or empty, skip this file
if (operation.startsWith("filter-") && (response.getBody() == null || response.getBody().length == 0)) {
// If the operation is filter and the response body is null or empty, skip this
// file
if (operation.startsWith("filter-")
&& (response.getBody() == null || response.getBody().length == 0)) {
logger.info("Skipping file due to failing {}", operation);
continue;
}
@ -362,7 +363,6 @@ public class PipelineController {
continue;
}
// Define filename
String filename;
if ("auto-rename".equals(operation)) {
@ -398,6 +398,79 @@ public class PipelineController {
outputFiles = newOutputFiles;
}
} else {
// Filter and collect all files that match the inputFileExtension
List<Resource> matchingFiles = outputFiles.stream()
.filter(file -> file.getFilename().endsWith(finalInputFileExtension))
.collect(Collectors.toList());
// Check if there are matching files
if (!matchingFiles.isEmpty()) {
// Create a new MultiValueMap for the request body
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
// Add all matching files to the body
for (Resource file : matchingFiles) {
body.add("fileInput", file);
}
// Add other parameters from the JSON node
Iterator<Map.Entry<String, JsonNode>> parameters = parametersNode.fields();
while (parameters.hasNext()) {
Map.Entry<String, JsonNode> parameter = parameters.next();
body.add(parameter.getKey(), parameter.getValue().asText());
}
// Set up headers, including API key
HttpHeaders headers = new HttpHeaders();
String apiKey = getApiKeyForUser();
headers.add("X-API-Key", apiKey);
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
// Create HttpEntity with the body and headers
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
// Make the request to the REST endpoint
ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
// Handle the response
if (response.getStatusCode().equals(HttpStatus.OK)) {
// Define filename
String filename;
if ("auto-rename".equals(operation)) {
// If the operation is "auto-rename", generate a new filename.
// This is a simple example of generating a filename using current timestamp.
// Modify as per your needs.
filename = "file_" + System.currentTimeMillis();
} else {
// Otherwise, keep the original filename.
filename = matchingFiles.get(0).getFilename();
}
// Check if the response body is a zip file
if (isZip(response.getBody())) {
// Unzip the file and add all the files to the new output files
newOutputFiles.addAll(unzip(response.getBody()));
} else {
Resource outputResource = new ByteArrayResource(response.getBody()) {
@Override
public String getFilename() {
return filename;
}
};
newOutputFiles.add(outputResource);
}
} else {
// Log error if the response status is not OK
logPrintStream.println("Error in multi-input operation: " + response.getBody());
hasErrors = true;
}
} else {
logPrintStream.println("No files with extension " + inputFileExtension + " found for multi-input operation " + operation);
hasErrors = true;
}
}
logPrintStream.close();
}
@ -408,14 +481,13 @@ public class PipelineController {
}
List<Resource> handleFiles(File[] files, String jsonString) throws Exception {
if(files == null || files.length == 0) {
if (files == null || files.length == 0) {
logger.info("No files");
return null;
}
logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length());
List<Resource> outputFiles = new ArrayList<>();
for (File file : files) {
@ -439,7 +511,7 @@ public class PipelineController {
}
List<Resource> handleFiles(MultipartFile[] files, String jsonString) throws Exception {
if(files == null || files.length == 0) {
if (files == null || files.length == 0) {
logger.info("No files");
return null;
}
@ -467,13 +539,13 @@ public class PipelineController {
@PostMapping("/handleData")
public ResponseEntity<byte[]> handleData(@ModelAttribute HandleDataRequest request) {
if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
return new ResponseEntity<>(HttpStatus.BAD_REQUEST);
}
MultipartFile[] files = request.getFileInput();
String jsonString = request.getJson();
if(files == null) {
if (files == null) {
return null;
}
logger.info("Received POST request to /handleData with {} files", files.length);

6
src/main/java/stirling/software/SPDF/model/ApiEndpoint.java
View file

@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode;
public class ApiEndpoint {
private String name;
private Map<String, JsonNode> parameters;
private String description;
public ApiEndpoint(String name, JsonNode postNode) {
this.name = name;
@ -16,6 +17,7 @@ public class ApiEndpoint {
String paramName = paramNode.path("name").asText();
parameters.put(paramName, paramNode);
});
this.description = postNode.path("description").asText();
}
public boolean areParametersValid(Map<String, Object> providedParams) {
@ -27,6 +29,10 @@ public class ApiEndpoint {
return true;
}
public String getDescription() {
return description;
}
@Override
public String toString() {
return "ApiEndpoint [name=" + name + ", parameters=" + parameters + "]";