pipeline refactor beginnings

src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java

@@ -1,6 +1,8 @@
 package stirling.software.SPDF.controller.api.pipeline;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
@@ -41,6 +43,9 @@ public class ApiDocService {
 		return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
 	}
 	
+	JsonNode apiDocsJsonRootNode;
+	
+	
 	//@EventListener(ApplicationReadyEvent.class)
 	private synchronized void loadApiDocumentation() {
         try {
@@ -56,9 +61,9 @@ public class ApiDocService {
             String apiDocsJson = response.getBody();
 
             ObjectMapper mapper = new ObjectMapper();
-            JsonNode root = mapper.readTree(apiDocsJson);
+            apiDocsJsonRootNode = mapper.readTree(apiDocsJson);
 
-            JsonNode paths = root.path("paths");
+            JsonNode paths = apiDocsJsonRootNode.path("paths");
             paths.fields().forEachRemaining(entry -> {
                 String path = entry.getKey();
                 JsonNode pathNode = entry.getValue();
@@ -84,6 +89,27 @@ public class ApiDocService {
         ApiEndpoint endpoint = apiDocumentation.get(operationName);
         return endpoint.areParametersValid(parameters);
     }
+    
+    public boolean isMultiInput(String operationName) {
+    	if(apiDocsJsonRootNode == null || apiDocumentation.size() == 0) {
+    		loadApiDocumentation();
+    	}
+    	if (!apiDocumentation.containsKey(operationName)) {
+            return false;
+        }
+
+        ApiEndpoint endpoint = apiDocumentation.get(operationName);
+        String description = endpoint.getDescription(); 
+
+        Pattern pattern = Pattern.compile("Type:(\\w+)");
+        Matcher matcher = pattern.matcher(description);
+        if (matcher.find()) {
+            String type = matcher.group(1);
+            return type.startsWith("MI");
+        }
+
+        return false;    	
+    }
 }
 
 // Model class for API Endpoint

src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java

@@ -15,6 +15,7 @@ import java.time.LocalDate;
 import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -69,16 +70,13 @@ public class PipelineController {
 
 	final String watchedFoldersDir = "./pipeline/watchedFolders/";
 	final String finishedFoldersDir = "./pipeline/finishedFolders/";
-	
+
 	@Autowired
 	private ApiDocService apiDocService;
-	
 
-	
-	
 	@Scheduled(fixedRate = 60000)
 	public void scanFolders() {
-		if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
+		if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
 			return;
 		}
 		Path watchedFolderPath = Paths.get(watchedFoldersDir);
@@ -106,26 +104,6 @@ public class PipelineController {
 		}
 	}
 
-	@Autowired
-	ApplicationProperties applicationProperties;
-	
-	@Autowired(required=false)
-	private UserServiceInterface userService;
-
-	private String getApiKeyForUser() {
-		if(userService == null)
-			return "";
-		return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
-	}
-	
-	 @Autowired
-	    private ServletContext servletContext;
-
-	    private String getBaseUrl() {
-	        String contextPath = servletContext.getContextPath();
-	        return "http://localhost:8080" + contextPath + "/";
-	    }
-	    
 	private void handleDirectory(Path dir) throws Exception {
 		logger.info("Handling directory: {}", dir);
 
@@ -134,20 +112,16 @@ public class PipelineController {
 			Files.createDirectory(processingDir);
 			logger.info("Created processing directory: {}", processingDir);
 		}
-		
+
 		Path jsonFile;
 		Optional<Path> jsonFileOptional;
 		// Find any JSON file in the directory
-	    try (Stream<Path> paths = Files.list(dir)) {
+		try (Stream<Path> paths = Files.list(dir)) {
-	    	jsonFileOptional = paths
+			jsonFileOptional = paths.filter(file -> file.toString().endsWith(".json")).findFirst();
-	                .filter(file -> file.toString().endsWith(".json"))
+		}
-	                .findFirst();
+
-	    }
+		if (jsonFileOptional.isPresent()) {
-	            
+			jsonFile = jsonFileOptional.get();
-	            
-		
-	    if (jsonFileOptional.isPresent()) {
-            jsonFile = jsonFileOptional.get();
 			// Read JSON file
 			String jsonString;
 			try {
@@ -175,21 +149,19 @@ public class PipelineController {
 			// For each operation in the pipeline
 			for (PipelineOperation operation : config.getOperations()) {
 				if (!apiDocService.isValidOperation(operation.getOperation(), operation.getParameters())) {
-		            logger.error("Invalid operation: " + operation.getOperation());
+					logger.error("Invalid operation: " + operation.getOperation());
-		            // Handle invalid operation
+					// Handle invalid operation
-		            throw new Exception("Invalid operation: " + operation.getOperation());
+					throw new Exception("Invalid operation: " + operation.getOperation());
-		        }
+				}
 				// Collect all files based on fileInput
 				File[] files;
 				String fileInput = (String) operation.getParameters().get("fileInput");
 				if ("automated".equals(fileInput)) {
 					// If fileInput is "automated", process all files in the directory
 					try (Stream<Path> paths = Files.list(dir)) {
-						files = paths
+						files = paths.filter(path -> !Files.isDirectory(path)) // exclude directories
-							    .filter(path -> !Files.isDirectory(path)) // exclude directories
+								.filter(path -> !path.equals(jsonFile)) // exclude jsonFile
-							    .filter(path -> !path.equals(jsonFile)) // exclude jsonFile
+								.map(Path::toFile).toArray(File[]::new);
-							    .map(Path::toFile)
-							    .toArray(File[]::new);
 
 					} catch (IOException e) {
 						e.printStackTrace();
@@ -203,33 +175,33 @@ public class PipelineController {
 				// Prepare the files for processing
 				List<File> filesToProcess = new ArrayList<>();
 				for (File file : files) {
-				    logger.info(file.getName());
+					logger.info(file.getName());
-				    logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName()));
+					logger.info("{} to {}", file.toPath(), processingDir.resolve(file.getName()));
-				    Files.move(file.toPath(), processingDir.resolve(file.getName()));
+					Files.move(file.toPath(), processingDir.resolve(file.getName()));
-				    filesToProcess.add(processingDir.resolve(file.getName()).toFile());
+					filesToProcess.add(processingDir.resolve(file.getName()).toFile());
 				}
 
 				// Process the files
 				try {
-				    List<Resource> resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString);
+					List<Resource> resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString);
-				    
+
-					if(resources == null) {
+					if (resources == null) {
 						return;
 					}
 					// Move resultant files and rename them as per config in JSON file
 					for (Resource resource : resources) {
 						String resourceName = resource.getFilename();
 						String baseName = resourceName.substring(0, resourceName.lastIndexOf("."));
-						String extension = resourceName.substring(resourceName.lastIndexOf(".")+1);
+						String extension = resourceName.substring(resourceName.lastIndexOf(".") + 1);
-						
+
 						String outputFileName = config.getOutputPattern().replace("{filename}", baseName);
-						
+
 						outputFileName = outputFileName.replace("{pipelineName}", config.getName());
 						DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");
 						outputFileName = outputFileName.replace("{date}", LocalDate.now().format(dateFormatter));
 						DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HHmmss");
 						outputFileName = outputFileName.replace("{time}", LocalTime.now().format(timeFormatter));
-						
+
 						outputFileName += "." + extension;
 						// {filename} {folder} {date} {tmime} {pipeline}
 						String outputDir = config.getOutputDir();
@@ -237,8 +209,8 @@ public class PipelineController {
 						String outputFolder = applicationProperties.getAutoPipeline().getOutputFolder();
 
 						if (outputFolder == null || outputFolder.isEmpty()) {
-						    // If the environment variable is not set, use the default value
+							// If the environment variable is not set, use the default value
-						    outputFolder = finishedFoldersDir;
+							outputFolder = finishedFoldersDir;
 						}
 						logger.info("outputDir 0={}", outputDir);
 						// Replace the placeholders in the outputDir string
@@ -249,19 +221,19 @@ public class PipelineController {
 						outputDir = outputDir.replace("//watchedFolders", "");
 						outputDir = outputDir.replace("\\\\watchedFolders", "");
 						outputDir = outputDir.replace("/watchedFolders", "");
-						
+
-						Path outputPath; 
+						Path outputPath;
 						logger.info("outputDir 2={}", outputDir);
 						if (Paths.get(outputDir).isAbsolute()) {
-						    // If it's an absolute path, use it directly
+							// If it's an absolute path, use it directly
-						    outputPath = Paths.get(outputDir);
+							outputPath = Paths.get(outputDir);
 						} else {
-						    // If it's a relative path, make it relative to the current working directory
+							// If it's a relative path, make it relative to the current working directory
-						    outputPath = Paths.get(".", outputDir);
+							outputPath = Paths.get(".", outputDir);
 						}
-						
+
 						logger.info("outputPath={}", outputPath);
-						
+
 						if (!Files.exists(outputPath)) {
 							try {
 								Files.createDirectories(outputPath);
@@ -271,13 +243,14 @@ public class PipelineController {
 								return;
 							}
 						}
-						logger.info("outputPath {}", outputPath);	
+						logger.info("outputPath {}", outputPath);
-						logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString());	
+						logger.info("outputPath.resolve(outputFileName).toString() {}",
+								outputPath.resolve(outputFileName).toString());
 						File newFile = new File(outputPath.resolve(outputFileName).toString());
-					    OutputStream os = new FileOutputStream(newFile);
+						OutputStream os = new FileOutputStream(newFile);
-					    os.write(((ByteArrayResource)resource).getByteArray());
+						os.write(((ByteArrayResource) resource).getByteArray());
-					    os.close();
+						os.close();
-						logger.info("made {}", outputPath.resolve(outputFileName));	
+						logger.info("made {}", outputPath.resolve(outputFileName));
 					}
 
 					// If successful, delete the original files
@@ -297,11 +270,31 @@ public class PipelineController {
 		}
 	}
 
+	@Autowired
+	ApplicationProperties applicationProperties;
+
+	@Autowired(required = false)
+	private UserServiceInterface userService;
+
+	private String getApiKeyForUser() {
+		if (userService == null)
+			return "";
+		return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
+	}
+
+	@Autowired
+	private ServletContext servletContext;
+
+	private String getBaseUrl() {
+		String contextPath = servletContext.getContextPath();
+		return "http://localhost:8080" + contextPath + "/";
+	}
+
 	List<Resource> processFiles(List<Resource> outputFiles, String jsonString) throws Exception {
-		
+
 		ObjectMapper mapper = new ObjectMapper();
 		logger.info("Running jsonString {}", jsonString);
-		
+
 		JsonNode jsonNode = mapper.readTree(jsonString);
 		logger.info("Running jsonNode {}", jsonNode);
 		JsonNode pipelineNode = jsonNode.get("pipeline");
@@ -313,7 +306,9 @@ public class PipelineController {
 
 		for (JsonNode operationNode : pipelineNode) {
 			String operation = operationNode.get("operation").asText();
-			logger.info("Running operation: {}", operation);
+			boolean isMultiInputOperation = apiDocService.isMultiInput(operation);
+
+			logger.info("Running operation: {} isMultiInputOperation {}", operation, isMultiInputOperation);
 			JsonNode parametersNode = operationNode.get("parameters");
 			String inputFileExtension = "";
 			if (operationNode.has("inputFileType")) {
@@ -321,82 +316,160 @@ public class PipelineController {
 			} else {
 				inputFileExtension = ".pdf";
 			}
-
+			final String finalInputFileExtension = inputFileExtension;
+			RestTemplate restTemplate = new RestTemplate();
+			String url = getBaseUrl() + operation;
+			
 			List<Resource> newOutputFiles = new ArrayList<>();
-			boolean hasInputFileType = false;
+			if (!isMultiInputOperation) {
+				
+				
 
-			for (Resource file : outputFiles) {
+				for (Resource file : outputFiles) {
-				if (file.getFilename().endsWith(inputFileExtension)) {
+					boolean hasInputFileType = false;
-					hasInputFileType = true;
+					if (file.getFilename().endsWith(inputFileExtension)) {
-					MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+						hasInputFileType = true;
-					body.add("fileInput", file);
+						MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+						body.add("fileInput", file);
 
-					Iterator<Map.Entry<String, JsonNode>> parameters = parametersNode.fields();
+						Iterator<Map.Entry<String, JsonNode>> parameters = parametersNode.fields();
-					while (parameters.hasNext()) {
+						while (parameters.hasNext()) {
-						Map.Entry<String, JsonNode> parameter = parameters.next();
+							Map.Entry<String, JsonNode> parameter = parameters.next();
-						body.add(parameter.getKey(), parameter.getValue().asText());
+							body.add(parameter.getKey(), parameter.getValue().asText());
+						}
+
+						HttpHeaders headers = new HttpHeaders();
+
+						String apiKey = getApiKeyForUser();
+						headers.add("X-API-Key", apiKey);
+						headers.setContentType(MediaType.MULTIPART_FORM_DATA);
+
+						HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
+
+						ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity,
+								byte[].class);
+
+						// If the operation is filter and the response body is null or empty, skip this
+						// file
+						if (operation.startsWith("filter-")
+								&& (response.getBody() == null || response.getBody().length == 0)) {
+							logger.info("Skipping file due to failing {}", operation);
+							continue;
+						}
+
+						if (!response.getStatusCode().equals(HttpStatus.OK)) {
+							logPrintStream.println("Error: " + response.getBody());
+							hasErrors = true;
+							continue;
+						}
+
+						// Define filename
+						String filename;
+						if ("auto-rename".equals(operation)) {
+							// If the operation is "auto-rename", generate a new filename.
+							// This is a simple example of generating a filename using current timestamp.
+							// Modify as per your needs.
+							filename = "file_" + System.currentTimeMillis();
+						} else {
+							// Otherwise, keep the original filename.
+							filename = file.getFilename();
+						}
+
+						// Check if the response body is a zip file
+						if (isZip(response.getBody())) {
+							// Unzip the file and add all the files to the new output files
+							newOutputFiles.addAll(unzip(response.getBody()));
+						} else {
+							Resource outputResource = new ByteArrayResource(response.getBody()) {
+								@Override
+								public String getFilename() {
+									return filename;
+								}
+							};
+							newOutputFiles.add(outputResource);
+						}
 					}
 
-					HttpHeaders headers = new HttpHeaders();
+					if (!hasInputFileType) {
-					
+						logPrintStream.println(
-					String apiKey = getApiKeyForUser();
+								"No files with extension " + inputFileExtension + " found for operation " + operation);
-					headers.add("X-API-Key", apiKey);
-					headers.setContentType(MediaType.MULTIPART_FORM_DATA);
-
-					HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
-
-					RestTemplate restTemplate = new RestTemplate();
-					String url = getBaseUrl() + operation;
-
-					ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
-
-					// If the operation is filter and the response body is null or empty, skip this file
-	                if (operation.startsWith("filter-") && (response.getBody() == null || response.getBody().length == 0)) {
-	                	logger.info("Skipping file due to failing {}", operation);
-	                    continue;
-	                }
-	                
-					if (!response.getStatusCode().equals(HttpStatus.OK)) {
-						logPrintStream.println("Error: " + response.getBody());
 						hasErrors = true;
-						continue;
 					}
-					
-					
-					// Define filename
-	                String filename;
-	                if ("auto-rename".equals(operation)) {
-	                    // If the operation is "auto-rename", generate a new filename.
-	                    // This is a simple example of generating a filename using current timestamp.
-	                    // Modify as per your needs.
-	                    filename = "file_" + System.currentTimeMillis();
-	                } else {
-	                    // Otherwise, keep the original filename.
-	                    filename = file.getFilename();
-	                }
 
-	                // Check if the response body is a zip file
+					outputFiles = newOutputFiles;
-	                if (isZip(response.getBody())) {
-	                    // Unzip the file and add all the files to the new output files
-	                    newOutputFiles.addAll(unzip(response.getBody()));
-	                } else {
-	                    Resource outputResource = new ByteArrayResource(response.getBody()) {
-	                        @Override
-	                        public String getFilename() {
-	                            return filename;
-	                        }
-	                    };
-	                    newOutputFiles.add(outputResource);
-	                }
 				}
 
-				if (!hasInputFileType) {
+			} else {
-					logPrintStream.println(
+				 // Filter and collect all files that match the inputFileExtension
-							"No files with extension " + inputFileExtension + " found for operation " + operation);
+			    List<Resource> matchingFiles = outputFiles.stream()
-					hasErrors = true;
+			                                              .filter(file -> file.getFilename().endsWith(finalInputFileExtension))
-				}
+			                                              .collect(Collectors.toList());
 
-				outputFiles = newOutputFiles;
+			    // Check if there are matching files
+			    if (!matchingFiles.isEmpty()) {
+			        // Create a new MultiValueMap for the request body
+			        MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+
+			        // Add all matching files to the body
+			        for (Resource file : matchingFiles) {
+			            body.add("fileInput", file);
+			        }
+
+			        // Add other parameters from the JSON node
+			        Iterator<Map.Entry<String, JsonNode>> parameters = parametersNode.fields();
+			        while (parameters.hasNext()) {
+			            Map.Entry<String, JsonNode> parameter = parameters.next();
+			            body.add(parameter.getKey(), parameter.getValue().asText());
+			        }
+
+			        // Set up headers, including API key
+			        HttpHeaders headers = new HttpHeaders();
+			        String apiKey = getApiKeyForUser();
+			        headers.add("X-API-Key", apiKey);
+			        headers.setContentType(MediaType.MULTIPART_FORM_DATA);
+
+			        // Create HttpEntity with the body and headers
+			        HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
+
+			        // Make the request to the REST endpoint
+			        ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
+
+			        // Handle the response
+			        if (response.getStatusCode().equals(HttpStatus.OK)) {
+			        	// Define filename
+						String filename;
+						if ("auto-rename".equals(operation)) {
+							// If the operation is "auto-rename", generate a new filename.
+							// This is a simple example of generating a filename using current timestamp.
+							// Modify as per your needs.
+							filename = "file_" + System.currentTimeMillis();
+						} else {
+							// Otherwise, keep the original filename.
+							filename = matchingFiles.get(0).getFilename();
+						}
+
+						// Check if the response body is a zip file
+						if (isZip(response.getBody())) {
+							// Unzip the file and add all the files to the new output files
+							newOutputFiles.addAll(unzip(response.getBody()));
+						} else {
+							Resource outputResource = new ByteArrayResource(response.getBody()) {
+								@Override
+								public String getFilename() {
+									return filename;
+								}
+							};
+							newOutputFiles.add(outputResource);
+						}
+			        } else {
+			            // Log error if the response status is not OK
+			            logPrintStream.println("Error in multi-input operation: " + response.getBody());
+			            hasErrors = true;
+			        }
+			    } else {
+			        logPrintStream.println("No files with extension " + inputFileExtension + " found for multi-input operation " + operation);
+			        hasErrors = true;
+			    }
 			}
 			logPrintStream.close();
 
@@ -408,38 +481,37 @@ public class PipelineController {
 	}
 
 	List<Resource> handleFiles(File[] files, String jsonString) throws Exception {
-		if(files == null || files.length == 0) {
+		if (files == null || files.length == 0) {
 			logger.info("No files");
 			return null;
 		}
-			
+
 		logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length());
-		
+
-	
 		List<Resource> outputFiles = new ArrayList<>();
 
 		for (File file : files) {
-		    Path path = Paths.get(file.getAbsolutePath());
+			Path path = Paths.get(file.getAbsolutePath());
-		    System.out.println("Reading file: " + path); // debug statement
+			System.out.println("Reading file: " + path); // debug statement
-		    
+
-		    if (Files.exists(path)) {
+			if (Files.exists(path)) {
-		        Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) {
+				Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) {
-		            @Override
+					@Override
-		            public String getFilename() {
+					public String getFilename() {
-		                return file.getName();
+						return file.getName();
-		            }
+					}
-		        };
+				};
-		        outputFiles.add(fileResource);
+				outputFiles.add(fileResource);
-		    } else {
+			} else {
-		        System.out.println("File not found: " + path); // debug statement
+				System.out.println("File not found: " + path); // debug statement
-		    }
+			}
 		}
 		logger.info("Files successfully loaded. Starting processing...");
 		return processFiles(outputFiles, jsonString);
 	}
 
 	List<Resource> handleFiles(MultipartFile[] files, String jsonString) throws Exception {
-		if(files == null || files.length == 0) {
+		if (files == null || files.length == 0) {
 			logger.info("No files");
 			return null;
 		}
@@ -467,15 +539,15 @@ public class PipelineController {
 
 	@PostMapping("/handleData")
 	public ResponseEntity<byte[]> handleData(@ModelAttribute HandleDataRequest request) {
-		if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
+		if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
 			return new ResponseEntity<>(HttpStatus.BAD_REQUEST);
 		}
-		
+
-	    MultipartFile[] files = request.getFileInput();
+		MultipartFile[] files = request.getFileInput();
-	    String jsonString = request.getJson();
+		String jsonString = request.getJson();
-	    if(files == null) {
+		if (files == null) {
-	    	return null;
+			return null;
-	    }
+		}
 		logger.info("Received POST request to /handleData with {} files", files.length);
 		try {
 			List<Resource> outputFiles = handleFiles(files, jsonString);

src/main/java/stirling/software/SPDF/model/ApiEndpoint.java

@@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode;
 public class ApiEndpoint {
     private String name;
     private Map<String, JsonNode> parameters;
-
+    private String description;
+    
     public ApiEndpoint(String name, JsonNode postNode) {
         this.name = name;
         this.parameters = new HashMap<>();
@@ -16,6 +17,7 @@ public class ApiEndpoint {
             String paramName = paramNode.path("name").asText();
             parameters.put(paramName, paramNode);
         });
+        this.description = postNode.path("description").asText();
     }
 
     public boolean areParametersValid(Map<String, Object> providedParams) {
@@ -27,6 +29,10 @@ public class ApiEndpoint {
         return true;
     }
 
+    public String getDescription() {
+        return description;
+    }
+    
 	@Override
 	public String toString() {
 		return "ApiEndpoint [name=" + name + ", parameters=" + parameters + "]";