pipeline refactor beginnings

src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java

@@ -1,6 +1,8 @@
 package stirling.software.SPDF.controller.api.pipeline;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
@@ -41,6 +43,9 @@ public class ApiDocService {
 		return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
 	}
 	
+	JsonNode apiDocsJsonRootNode;
+	
+	
 	//@EventListener(ApplicationReadyEvent.class)
 	private synchronized void loadApiDocumentation() {
         try {
@@ -56,9 +61,9 @@ public class ApiDocService {
             String apiDocsJson = response.getBody();
 
             ObjectMapper mapper = new ObjectMapper();
-            JsonNode root = mapper.readTree(apiDocsJson);
+            apiDocsJsonRootNode = mapper.readTree(apiDocsJson);
 
-            JsonNode paths = root.path("paths");
+            JsonNode paths = apiDocsJsonRootNode.path("paths");
             paths.fields().forEachRemaining(entry -> {
                 String path = entry.getKey();
                 JsonNode pathNode = entry.getValue();
@@ -84,6 +89,27 @@ public class ApiDocService {
         ApiEndpoint endpoint = apiDocumentation.get(operationName);
         return endpoint.areParametersValid(parameters);
     }
+    
+    public boolean isMultiInput(String operationName) {
+    	if(apiDocsJsonRootNode == null || apiDocumentation.size() == 0) {
+    		loadApiDocumentation();
+    	}
+    	if (!apiDocumentation.containsKey(operationName)) {
+            return false;
+        }
+
+        ApiEndpoint endpoint = apiDocumentation.get(operationName);
+        String description = endpoint.getDescription(); 
+
+        Pattern pattern = Pattern.compile("Type:(\\w+)");
+        Matcher matcher = pattern.matcher(description);
+        if (matcher.find()) {
+            String type = matcher.group(1);
+            return type.startsWith("MI");
+        }
+
+        return false;    	
+    }
 }
 
 // Model class for API Endpoint

src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java

@@ -15,6 +15,7 @@ import java.time.LocalDate;
 import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -73,12 +74,9 @@ public class PipelineController {
 	@Autowired
 	private ApiDocService apiDocService;
 
-
-	
-	
 	@Scheduled(fixedRate = 60000)
 	public void scanFolders() {
-		if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
+		if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
 			return;
 		}
 		Path watchedFolderPath = Paths.get(watchedFoldersDir);
@@ -106,26 +104,6 @@ public class PipelineController {
 		}
 	}
 
-	@Autowired
-	ApplicationProperties applicationProperties;
-	
-	@Autowired(required=false)
-	private UserServiceInterface userService;
-
-	private String getApiKeyForUser() {
-		if(userService == null)
-			return "";
-		return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
-	}
-	
-	 @Autowired
-	    private ServletContext servletContext;
-
-	    private String getBaseUrl() {
-	        String contextPath = servletContext.getContextPath();
-	        return "http://localhost:8080" + contextPath + "/";
-	    }
-	    
 	private void handleDirectory(Path dir) throws Exception {
 		logger.info("Handling directory: {}", dir);
 
@@ -139,13 +117,9 @@ public class PipelineController {
 		Optional<Path> jsonFileOptional;
 		// Find any JSON file in the directory
 		try (Stream<Path> paths = Files.list(dir)) {
-	    	jsonFileOptional = paths
+			jsonFileOptional = paths.filter(file -> file.toString().endsWith(".json")).findFirst();
-	                .filter(file -> file.toString().endsWith(".json"))
-	                .findFirst();
 		}
 
-	            
-		
 		if (jsonFileOptional.isPresent()) {
 			jsonFile = jsonFileOptional.get();
 			// Read JSON file
@@ -185,11 +159,9 @@ public class PipelineController {
 				if ("automated".equals(fileInput)) {
 					// If fileInput is "automated", process all files in the directory
 					try (Stream<Path> paths = Files.list(dir)) {
-						files = paths
+						files = paths.filter(path -> !Files.isDirectory(path)) // exclude directories
-							    .filter(path -> !Files.isDirectory(path)) // exclude directories
 								.filter(path -> !path.equals(jsonFile)) // exclude jsonFile
-							    .map(Path::toFile)
+								.map(Path::toFile).toArray(File[]::new);
-							    .toArray(File[]::new);
 
 					} catch (IOException e) {
 						e.printStackTrace();
@@ -204,7 +176,7 @@ public class PipelineController {
 				List<File> filesToProcess = new ArrayList<>();
 				for (File file : files) {
 					logger.info(file.getName());
-				    logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName()));
+					logger.info("{} to {}", file.toPath(), processingDir.resolve(file.getName()));
 					Files.move(file.toPath(), processingDir.resolve(file.getName()));
 					filesToProcess.add(processingDir.resolve(file.getName()).toFile());
 				}
@@ -213,14 +185,14 @@ public class PipelineController {
 				try {
 					List<Resource> resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString);
 
-					if(resources == null) {
+					if (resources == null) {
 						return;
 					}
 					// Move resultant files and rename them as per config in JSON file
 					for (Resource resource : resources) {
 						String resourceName = resource.getFilename();
 						String baseName = resourceName.substring(0, resourceName.lastIndexOf("."));
-						String extension = resourceName.substring(resourceName.lastIndexOf(".")+1);
+						String extension = resourceName.substring(resourceName.lastIndexOf(".") + 1);
 
 						String outputFileName = config.getOutputPattern().replace("{filename}", baseName);
 
@@ -272,10 +244,11 @@ public class PipelineController {
 							}
 						}
 						logger.info("outputPath {}", outputPath);
-						logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString());	
+						logger.info("outputPath.resolve(outputFileName).toString() {}",
+								outputPath.resolve(outputFileName).toString());
 						File newFile = new File(outputPath.resolve(outputFileName).toString());
 						OutputStream os = new FileOutputStream(newFile);
-					    os.write(((ByteArrayResource)resource).getByteArray());
+						os.write(((ByteArrayResource) resource).getByteArray());
 						os.close();
 						logger.info("made {}", outputPath.resolve(outputFileName));
 					}
@@ -297,6 +270,26 @@ public class PipelineController {
 		}
 	}
 
+	@Autowired
+	ApplicationProperties applicationProperties;
+
+	@Autowired(required = false)
+	private UserServiceInterface userService;
+
+	private String getApiKeyForUser() {
+		if (userService == null)
+			return "";
+		return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
+	}
+
+	@Autowired
+	private ServletContext servletContext;
+
+	private String getBaseUrl() {
+		String contextPath = servletContext.getContextPath();
+		return "http://localhost:8080" + contextPath + "/";
+	}
+
 	List<Resource> processFiles(List<Resource> outputFiles, String jsonString) throws Exception {
 
 		ObjectMapper mapper = new ObjectMapper();
@@ -313,7 +306,9 @@ public class PipelineController {
 
 		for (JsonNode operationNode : pipelineNode) {
 			String operation = operationNode.get("operation").asText();
-			logger.info("Running operation: {}", operation);
+			boolean isMultiInputOperation = apiDocService.isMultiInput(operation);
+
+			logger.info("Running operation: {} isMultiInputOperation {}", operation, isMultiInputOperation);
 			JsonNode parametersNode = operationNode.get("parameters");
 			String inputFileExtension = "";
 			if (operationNode.has("inputFileType")) {
@@ -321,11 +316,17 @@ public class PipelineController {
 			} else {
 				inputFileExtension = ".pdf";
 			}
+			final String finalInputFileExtension = inputFileExtension;
+			RestTemplate restTemplate = new RestTemplate();
+			String url = getBaseUrl() + operation;
 			
 			List<Resource> newOutputFiles = new ArrayList<>();
-			boolean hasInputFileType = false;
+			if (!isMultiInputOperation) {
+				
+				
 
 				for (Resource file : outputFiles) {
+					boolean hasInputFileType = false;
 					if (file.getFilename().endsWith(inputFileExtension)) {
 						hasInputFileType = true;
 						MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
@@ -345,13 +346,13 @@ public class PipelineController {
 
 						HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
 
-					RestTemplate restTemplate = new RestTemplate();
+						ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity,
-					String url = getBaseUrl() + operation;
+								byte[].class);
 
-					ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
+						// If the operation is filter and the response body is null or empty, skip this
-
+						// file
-					// If the operation is filter and the response body is null or empty, skip this file
+						if (operation.startsWith("filter-")
-	                if (operation.startsWith("filter-") && (response.getBody() == null || response.getBody().length == 0)) {
+								&& (response.getBody() == null || response.getBody().length == 0)) {
 							logger.info("Skipping file due to failing {}", operation);
 							continue;
 						}
@@ -362,7 +363,6 @@ public class PipelineController {
 							continue;
 						}
 
-					
 						// Define filename
 						String filename;
 						if ("auto-rename".equals(operation)) {
@@ -398,6 +398,79 @@ public class PipelineController {
 
 					outputFiles = newOutputFiles;
 				}
+
+			} else {
+				 // Filter and collect all files that match the inputFileExtension
+			    List<Resource> matchingFiles = outputFiles.stream()
+			                                              .filter(file -> file.getFilename().endsWith(finalInputFileExtension))
+			                                              .collect(Collectors.toList());
+
+			    // Check if there are matching files
+			    if (!matchingFiles.isEmpty()) {
+			        // Create a new MultiValueMap for the request body
+			        MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+
+			        // Add all matching files to the body
+			        for (Resource file : matchingFiles) {
+			            body.add("fileInput", file);
+			        }
+
+			        // Add other parameters from the JSON node
+			        Iterator<Map.Entry<String, JsonNode>> parameters = parametersNode.fields();
+			        while (parameters.hasNext()) {
+			            Map.Entry<String, JsonNode> parameter = parameters.next();
+			            body.add(parameter.getKey(), parameter.getValue().asText());
+			        }
+
+			        // Set up headers, including API key
+			        HttpHeaders headers = new HttpHeaders();
+			        String apiKey = getApiKeyForUser();
+			        headers.add("X-API-Key", apiKey);
+			        headers.setContentType(MediaType.MULTIPART_FORM_DATA);
+
+			        // Create HttpEntity with the body and headers
+			        HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
+
+			        // Make the request to the REST endpoint
+			        ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
+
+			        // Handle the response
+			        if (response.getStatusCode().equals(HttpStatus.OK)) {
+			        	// Define filename
+						String filename;
+						if ("auto-rename".equals(operation)) {
+							// If the operation is "auto-rename", generate a new filename.
+							// This is a simple example of generating a filename using current timestamp.
+							// Modify as per your needs.
+							filename = "file_" + System.currentTimeMillis();
+						} else {
+							// Otherwise, keep the original filename.
+							filename = matchingFiles.get(0).getFilename();
+						}
+
+						// Check if the response body is a zip file
+						if (isZip(response.getBody())) {
+							// Unzip the file and add all the files to the new output files
+							newOutputFiles.addAll(unzip(response.getBody()));
+						} else {
+							Resource outputResource = new ByteArrayResource(response.getBody()) {
+								@Override
+								public String getFilename() {
+									return filename;
+								}
+							};
+							newOutputFiles.add(outputResource);
+						}
+			        } else {
+			            // Log error if the response status is not OK
+			            logPrintStream.println("Error in multi-input operation: " + response.getBody());
+			            hasErrors = true;
+			        }
+			    } else {
+			        logPrintStream.println("No files with extension " + inputFileExtension + " found for multi-input operation " + operation);
+			        hasErrors = true;
+			    }
+			}
 			logPrintStream.close();
 
 		}
@@ -408,14 +481,13 @@ public class PipelineController {
 	}
 
 	List<Resource> handleFiles(File[] files, String jsonString) throws Exception {
-		if(files == null || files.length == 0) {
+		if (files == null || files.length == 0) {
 			logger.info("No files");
 			return null;
 		}
 
 		logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length());
 
-	
 		List<Resource> outputFiles = new ArrayList<>();
 
 		for (File file : files) {
@@ -439,7 +511,7 @@ public class PipelineController {
 	}
 
 	List<Resource> handleFiles(MultipartFile[] files, String jsonString) throws Exception {
-		if(files == null || files.length == 0) {
+		if (files == null || files.length == 0) {
 			logger.info("No files");
 			return null;
 		}
@@ -467,13 +539,13 @@ public class PipelineController {
 
 	@PostMapping("/handleData")
 	public ResponseEntity<byte[]> handleData(@ModelAttribute HandleDataRequest request) {
-		if(!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
+		if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
 			return new ResponseEntity<>(HttpStatus.BAD_REQUEST);
 		}
 
 		MultipartFile[] files = request.getFileInput();
 		String jsonString = request.getJson();
-	    if(files == null) {
+		if (files == null) {
 			return null;
 		}
 		logger.info("Received POST request to /handleData with {} files", files.length);

src/main/java/stirling/software/SPDF/model/ApiEndpoint.java

@@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode;
 public class ApiEndpoint {
     private String name;
     private Map<String, JsonNode> parameters;
+    private String description;
     
     public ApiEndpoint(String name, JsonNode postNode) {
         this.name = name;
@@ -16,6 +17,7 @@ public class ApiEndpoint {
             String paramName = paramNode.path("name").asText();
             parameters.put(paramName, paramNode);
         });
+        this.description = postNode.path("description").asText();
     }
 
     public boolean areParametersValid(Map<String, Object> providedParams) {
@@ -27,6 +29,10 @@ public class ApiEndpoint {
         return true;
     }
 
+    public String getDescription() {
+        return description;
+    }
+    
 	@Override
 	public String toString() {
 		return "ApiEndpoint [name=" + name + ", parameters=" + parameters + "]";