From 5fcb4e893b7ff3fa8da96475218f35eb74bdd1fa Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Thu, 28 Dec 2023 17:23:19 +0000 Subject: [PATCH] pipeline refactor beginnings --- .../api/pipeline/ApiDocService.java | 30 +- .../api/pipeline/PipelineController.java | 404 +++++++++++------- .../software/SPDF/model/ApiEndpoint.java | 8 +- 3 files changed, 273 insertions(+), 169 deletions(-) diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java index 8ceaaa76..cfe2fea3 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java @@ -1,6 +1,8 @@ package stirling.software.SPDF.controller.api.pipeline; import java.util.HashMap; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -41,6 +43,9 @@ public class ApiDocService { return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId()); } + JsonNode apiDocsJsonRootNode; + + //@EventListener(ApplicationReadyEvent.class) private synchronized void loadApiDocumentation() { try { @@ -56,9 +61,9 @@ public class ApiDocService { String apiDocsJson = response.getBody(); ObjectMapper mapper = new ObjectMapper(); - JsonNode root = mapper.readTree(apiDocsJson); + apiDocsJsonRootNode = mapper.readTree(apiDocsJson); - JsonNode paths = root.path("paths"); + JsonNode paths = apiDocsJsonRootNode.path("paths"); paths.fields().forEachRemaining(entry -> { String path = entry.getKey(); JsonNode pathNode = entry.getValue(); @@ -84,6 +89,27 @@ public class ApiDocService { ApiEndpoint endpoint = apiDocumentation.get(operationName); return endpoint.areParametersValid(parameters); } + + public boolean isMultiInput(String operationName) { + if(apiDocsJsonRootNode == null || apiDocumentation.size() == 0) { + loadApiDocumentation(); + } + if (!apiDocumentation.containsKey(operationName)) { + return false; + } + + ApiEndpoint endpoint = apiDocumentation.get(operationName); + String description = endpoint.getDescription(); + + Pattern pattern = Pattern.compile("Type:(\\w+)"); + Matcher matcher = pattern.matcher(description); + if (matcher.find()) { + String type = matcher.group(1); + return type.startsWith("MI"); + } + + return false; + } } // Model class for API Endpoint diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java index 47954026..2f76cd77 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java @@ -15,6 +15,7 @@ import java.time.LocalDate; import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -69,16 +70,13 @@ public class PipelineController { final String watchedFoldersDir = "./pipeline/watchedFolders/"; final String finishedFoldersDir = "./pipeline/finishedFolders/"; - + @Autowired private ApiDocService apiDocService; - - - @Scheduled(fixedRate = 60000) public void scanFolders() { - if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) { + if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) { return; } Path watchedFolderPath = Paths.get(watchedFoldersDir); @@ -106,26 +104,6 @@ public class PipelineController { } } - @Autowired - ApplicationProperties applicationProperties; - - @Autowired(required=false) - private UserServiceInterface userService; - - private String getApiKeyForUser() { - if(userService == null) - return ""; - return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId()); - } - - @Autowired - private ServletContext servletContext; - - private String getBaseUrl() { - String contextPath = servletContext.getContextPath(); - return "http://localhost:8080" + contextPath + "/"; - } - private void handleDirectory(Path dir) throws Exception { logger.info("Handling directory: {}", dir); @@ -134,20 +112,16 @@ public class PipelineController { Files.createDirectory(processingDir); logger.info("Created processing directory: {}", processingDir); } - + Path jsonFile; Optional jsonFileOptional; // Find any JSON file in the directory - try (Stream paths = Files.list(dir)) { - jsonFileOptional = paths - .filter(file -> file.toString().endsWith(".json")) - .findFirst(); - } - - - - if (jsonFileOptional.isPresent()) { - jsonFile = jsonFileOptional.get(); + try (Stream paths = Files.list(dir)) { + jsonFileOptional = paths.filter(file -> file.toString().endsWith(".json")).findFirst(); + } + + if (jsonFileOptional.isPresent()) { + jsonFile = jsonFileOptional.get(); // Read JSON file String jsonString; try { @@ -175,21 +149,19 @@ public class PipelineController { // For each operation in the pipeline for (PipelineOperation operation : config.getOperations()) { if (!apiDocService.isValidOperation(operation.getOperation(), operation.getParameters())) { - logger.error("Invalid operation: " + operation.getOperation()); - // Handle invalid operation - throw new Exception("Invalid operation: " + operation.getOperation()); - } + logger.error("Invalid operation: " + operation.getOperation()); + // Handle invalid operation + throw new Exception("Invalid operation: " + operation.getOperation()); + } // Collect all files based on fileInput File[] files; String fileInput = (String) operation.getParameters().get("fileInput"); if ("automated".equals(fileInput)) { // If fileInput is "automated", process all files in the directory try (Stream paths = Files.list(dir)) { - files = paths - .filter(path -> !Files.isDirectory(path)) // exclude directories - .filter(path -> !path.equals(jsonFile)) // exclude jsonFile - .map(Path::toFile) - .toArray(File[]::new); + files = paths.filter(path -> !Files.isDirectory(path)) // exclude directories + .filter(path -> !path.equals(jsonFile)) // exclude jsonFile + .map(Path::toFile).toArray(File[]::new); } catch (IOException e) { e.printStackTrace(); @@ -203,33 +175,33 @@ public class PipelineController { // Prepare the files for processing List filesToProcess = new ArrayList<>(); for (File file : files) { - logger.info(file.getName()); - logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName())); - Files.move(file.toPath(), processingDir.resolve(file.getName())); - filesToProcess.add(processingDir.resolve(file.getName()).toFile()); + logger.info(file.getName()); + logger.info("{} to {}", file.toPath(), processingDir.resolve(file.getName())); + Files.move(file.toPath(), processingDir.resolve(file.getName())); + filesToProcess.add(processingDir.resolve(file.getName()).toFile()); } // Process the files try { - List resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString); - - if(resources == null) { + List resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString); + + if (resources == null) { return; } // Move resultant files and rename them as per config in JSON file for (Resource resource : resources) { String resourceName = resource.getFilename(); String baseName = resourceName.substring(0, resourceName.lastIndexOf(".")); - String extension = resourceName.substring(resourceName.lastIndexOf(".")+1); - + String extension = resourceName.substring(resourceName.lastIndexOf(".") + 1); + String outputFileName = config.getOutputPattern().replace("{filename}", baseName); - + outputFileName = outputFileName.replace("{pipelineName}", config.getName()); DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd"); outputFileName = outputFileName.replace("{date}", LocalDate.now().format(dateFormatter)); DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HHmmss"); outputFileName = outputFileName.replace("{time}", LocalTime.now().format(timeFormatter)); - + outputFileName += "." + extension; // {filename} {folder} {date} {tmime} {pipeline} String outputDir = config.getOutputDir(); @@ -237,8 +209,8 @@ public class PipelineController { String outputFolder = applicationProperties.getAutoPipeline().getOutputFolder(); if (outputFolder == null || outputFolder.isEmpty()) { - // If the environment variable is not set, use the default value - outputFolder = finishedFoldersDir; + // If the environment variable is not set, use the default value + outputFolder = finishedFoldersDir; } logger.info("outputDir 0={}", outputDir); // Replace the placeholders in the outputDir string @@ -249,19 +221,19 @@ public class PipelineController { outputDir = outputDir.replace("//watchedFolders", ""); outputDir = outputDir.replace("\\\\watchedFolders", ""); outputDir = outputDir.replace("/watchedFolders", ""); - - Path outputPath; + + Path outputPath; logger.info("outputDir 2={}", outputDir); if (Paths.get(outputDir).isAbsolute()) { - // If it's an absolute path, use it directly - outputPath = Paths.get(outputDir); + // If it's an absolute path, use it directly + outputPath = Paths.get(outputDir); } else { - // If it's a relative path, make it relative to the current working directory - outputPath = Paths.get(".", outputDir); + // If it's a relative path, make it relative to the current working directory + outputPath = Paths.get(".", outputDir); } - + logger.info("outputPath={}", outputPath); - + if (!Files.exists(outputPath)) { try { Files.createDirectories(outputPath); @@ -271,13 +243,14 @@ public class PipelineController { return; } } - logger.info("outputPath {}", outputPath); - logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString()); + logger.info("outputPath {}", outputPath); + logger.info("outputPath.resolve(outputFileName).toString() {}", + outputPath.resolve(outputFileName).toString()); File newFile = new File(outputPath.resolve(outputFileName).toString()); - OutputStream os = new FileOutputStream(newFile); - os.write(((ByteArrayResource)resource).getByteArray()); - os.close(); - logger.info("made {}", outputPath.resolve(outputFileName)); + OutputStream os = new FileOutputStream(newFile); + os.write(((ByteArrayResource) resource).getByteArray()); + os.close(); + logger.info("made {}", outputPath.resolve(outputFileName)); } // If successful, delete the original files @@ -297,11 +270,31 @@ public class PipelineController { } } + @Autowired + ApplicationProperties applicationProperties; + + @Autowired(required = false) + private UserServiceInterface userService; + + private String getApiKeyForUser() { + if (userService == null) + return ""; + return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId()); + } + + @Autowired + private ServletContext servletContext; + + private String getBaseUrl() { + String contextPath = servletContext.getContextPath(); + return "http://localhost:8080" + contextPath + "/"; + } + List processFiles(List outputFiles, String jsonString) throws Exception { - + ObjectMapper mapper = new ObjectMapper(); logger.info("Running jsonString {}", jsonString); - + JsonNode jsonNode = mapper.readTree(jsonString); logger.info("Running jsonNode {}", jsonNode); JsonNode pipelineNode = jsonNode.get("pipeline"); @@ -313,7 +306,9 @@ public class PipelineController { for (JsonNode operationNode : pipelineNode) { String operation = operationNode.get("operation").asText(); - logger.info("Running operation: {}", operation); + boolean isMultiInputOperation = apiDocService.isMultiInput(operation); + + logger.info("Running operation: {} isMultiInputOperation {}", operation, isMultiInputOperation); JsonNode parametersNode = operationNode.get("parameters"); String inputFileExtension = ""; if (operationNode.has("inputFileType")) { @@ -321,82 +316,160 @@ public class PipelineController { } else { inputFileExtension = ".pdf"; } - + final String finalInputFileExtension = inputFileExtension; + RestTemplate restTemplate = new RestTemplate(); + String url = getBaseUrl() + operation; + List newOutputFiles = new ArrayList<>(); - boolean hasInputFileType = false; + if (!isMultiInputOperation) { + + - for (Resource file : outputFiles) { - if (file.getFilename().endsWith(inputFileExtension)) { - hasInputFileType = true; - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("fileInput", file); + for (Resource file : outputFiles) { + boolean hasInputFileType = false; + if (file.getFilename().endsWith(inputFileExtension)) { + hasInputFileType = true; + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("fileInput", file); - Iterator> parameters = parametersNode.fields(); - while (parameters.hasNext()) { - Map.Entry parameter = parameters.next(); - body.add(parameter.getKey(), parameter.getValue().asText()); + Iterator> parameters = parametersNode.fields(); + while (parameters.hasNext()) { + Map.Entry parameter = parameters.next(); + body.add(parameter.getKey(), parameter.getValue().asText()); + } + + HttpHeaders headers = new HttpHeaders(); + + String apiKey = getApiKeyForUser(); + headers.add("X-API-Key", apiKey); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + HttpEntity> entity = new HttpEntity<>(body, headers); + + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, + byte[].class); + + // If the operation is filter and the response body is null or empty, skip this + // file + if (operation.startsWith("filter-") + && (response.getBody() == null || response.getBody().length == 0)) { + logger.info("Skipping file due to failing {}", operation); + continue; + } + + if (!response.getStatusCode().equals(HttpStatus.OK)) { + logPrintStream.println("Error: " + response.getBody()); + hasErrors = true; + continue; + } + + // Define filename + String filename; + if ("auto-rename".equals(operation)) { + // If the operation is "auto-rename", generate a new filename. + // This is a simple example of generating a filename using current timestamp. + // Modify as per your needs. + filename = "file_" + System.currentTimeMillis(); + } else { + // Otherwise, keep the original filename. + filename = file.getFilename(); + } + + // Check if the response body is a zip file + if (isZip(response.getBody())) { + // Unzip the file and add all the files to the new output files + newOutputFiles.addAll(unzip(response.getBody())); + } else { + Resource outputResource = new ByteArrayResource(response.getBody()) { + @Override + public String getFilename() { + return filename; + } + }; + newOutputFiles.add(outputResource); + } } - HttpHeaders headers = new HttpHeaders(); - - String apiKey = getApiKeyForUser(); - headers.add("X-API-Key", apiKey); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - - HttpEntity> entity = new HttpEntity<>(body, headers); - - RestTemplate restTemplate = new RestTemplate(); - String url = getBaseUrl() + operation; - - ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class); - - // If the operation is filter and the response body is null or empty, skip this file - if (operation.startsWith("filter-") && (response.getBody() == null || response.getBody().length == 0)) { - logger.info("Skipping file due to failing {}", operation); - continue; - } - - if (!response.getStatusCode().equals(HttpStatus.OK)) { - logPrintStream.println("Error: " + response.getBody()); + if (!hasInputFileType) { + logPrintStream.println( + "No files with extension " + inputFileExtension + " found for operation " + operation); hasErrors = true; - continue; } - - - // Define filename - String filename; - if ("auto-rename".equals(operation)) { - // If the operation is "auto-rename", generate a new filename. - // This is a simple example of generating a filename using current timestamp. - // Modify as per your needs. - filename = "file_" + System.currentTimeMillis(); - } else { - // Otherwise, keep the original filename. - filename = file.getFilename(); - } - // Check if the response body is a zip file - if (isZip(response.getBody())) { - // Unzip the file and add all the files to the new output files - newOutputFiles.addAll(unzip(response.getBody())); - } else { - Resource outputResource = new ByteArrayResource(response.getBody()) { - @Override - public String getFilename() { - return filename; - } - }; - newOutputFiles.add(outputResource); - } + outputFiles = newOutputFiles; } - if (!hasInputFileType) { - logPrintStream.println( - "No files with extension " + inputFileExtension + " found for operation " + operation); - hasErrors = true; - } + } else { + // Filter and collect all files that match the inputFileExtension + List matchingFiles = outputFiles.stream() + .filter(file -> file.getFilename().endsWith(finalInputFileExtension)) + .collect(Collectors.toList()); - outputFiles = newOutputFiles; + // Check if there are matching files + if (!matchingFiles.isEmpty()) { + // Create a new MultiValueMap for the request body + MultiValueMap body = new LinkedMultiValueMap<>(); + + // Add all matching files to the body + for (Resource file : matchingFiles) { + body.add("fileInput", file); + } + + // Add other parameters from the JSON node + Iterator> parameters = parametersNode.fields(); + while (parameters.hasNext()) { + Map.Entry parameter = parameters.next(); + body.add(parameter.getKey(), parameter.getValue().asText()); + } + + // Set up headers, including API key + HttpHeaders headers = new HttpHeaders(); + String apiKey = getApiKeyForUser(); + headers.add("X-API-Key", apiKey); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + // Create HttpEntity with the body and headers + HttpEntity> entity = new HttpEntity<>(body, headers); + + // Make the request to the REST endpoint + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class); + + // Handle the response + if (response.getStatusCode().equals(HttpStatus.OK)) { + // Define filename + String filename; + if ("auto-rename".equals(operation)) { + // If the operation is "auto-rename", generate a new filename. + // This is a simple example of generating a filename using current timestamp. + // Modify as per your needs. + filename = "file_" + System.currentTimeMillis(); + } else { + // Otherwise, keep the original filename. + filename = matchingFiles.get(0).getFilename(); + } + + // Check if the response body is a zip file + if (isZip(response.getBody())) { + // Unzip the file and add all the files to the new output files + newOutputFiles.addAll(unzip(response.getBody())); + } else { + Resource outputResource = new ByteArrayResource(response.getBody()) { + @Override + public String getFilename() { + return filename; + } + }; + newOutputFiles.add(outputResource); + } + } else { + // Log error if the response status is not OK + logPrintStream.println("Error in multi-input operation: " + response.getBody()); + hasErrors = true; + } + } else { + logPrintStream.println("No files with extension " + inputFileExtension + " found for multi-input operation " + operation); + hasErrors = true; + } } logPrintStream.close(); @@ -408,38 +481,37 @@ public class PipelineController { } List handleFiles(File[] files, String jsonString) throws Exception { - if(files == null || files.length == 0) { + if (files == null || files.length == 0) { logger.info("No files"); return null; } - + logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); - - + List outputFiles = new ArrayList<>(); for (File file : files) { - Path path = Paths.get(file.getAbsolutePath()); - System.out.println("Reading file: " + path); // debug statement - - if (Files.exists(path)) { - Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) { - @Override - public String getFilename() { - return file.getName(); - } - }; - outputFiles.add(fileResource); - } else { - System.out.println("File not found: " + path); // debug statement - } + Path path = Paths.get(file.getAbsolutePath()); + System.out.println("Reading file: " + path); // debug statement + + if (Files.exists(path)) { + Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) { + @Override + public String getFilename() { + return file.getName(); + } + }; + outputFiles.add(fileResource); + } else { + System.out.println("File not found: " + path); // debug statement + } } logger.info("Files successfully loaded. Starting processing..."); return processFiles(outputFiles, jsonString); } List handleFiles(MultipartFile[] files, String jsonString) throws Exception { - if(files == null || files.length == 0) { + if (files == null || files.length == 0) { logger.info("No files"); return null; } @@ -467,15 +539,15 @@ public class PipelineController { @PostMapping("/handleData") public ResponseEntity handleData(@ModelAttribute HandleDataRequest request) { - if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) { + if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) { return new ResponseEntity<>(HttpStatus.BAD_REQUEST); } - - MultipartFile[] files = request.getFileInput(); - String jsonString = request.getJson(); - if(files == null) { - return null; - } + + MultipartFile[] files = request.getFileInput(); + String jsonString = request.getJson(); + if (files == null) { + return null; + } logger.info("Received POST request to /handleData with {} files", files.length); try { List outputFiles = handleFiles(files, jsonString); diff --git a/src/main/java/stirling/software/SPDF/model/ApiEndpoint.java b/src/main/java/stirling/software/SPDF/model/ApiEndpoint.java index d2a1de52..1838b763 100644 --- a/src/main/java/stirling/software/SPDF/model/ApiEndpoint.java +++ b/src/main/java/stirling/software/SPDF/model/ApiEndpoint.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; public class ApiEndpoint { private String name; private Map parameters; - + private String description; + public ApiEndpoint(String name, JsonNode postNode) { this.name = name; this.parameters = new HashMap<>(); @@ -16,6 +17,7 @@ public class ApiEndpoint { String paramName = paramNode.path("name").asText(); parameters.put(paramName, paramNode); }); + this.description = postNode.path("description").asText(); } public boolean areParametersValid(Map providedParams) { @@ -27,6 +29,10 @@ public class ApiEndpoint { return true; } + public String getDescription() { + return description; + } + @Override public String toString() { return "ApiEndpoint [name=" + name + ", parameters=" + parameters + "]";