pipeline

.gitignore

@@ -15,8 +15,8 @@ local.properties
 .classpath
 .project
 version.properties
-pipeline/
+pipeline/watchedFolders/
-
+pipeline/finishedFolders/
 #### Stirling-PDF Files ###
 customFiles/
 configs/

Dockerfile

@@ -18,13 +18,14 @@ ENV DOCKER_ENABLE_SECURITY=false \
 ##    mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME
 
 # Set up necessary directories and permissions
-RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles 
+RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs  /pipeline/watchedFolders /pipeline/finishedFolders
 ##&& \
 ##    chown -R stirlingpdfuser:stirlingpdfgroup /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles && \
 ##    chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/tesseract-ocr-original
 
 # Copy necessary files
 COPY ./scripts/* /scripts/
+COPY ./pipeline/ /pipeline/
 COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
 COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/
 COPY build/libs/*.jar app.jar

Dockerfile-lite

@@ -28,13 +28,14 @@ ENV DOCKER_ENABLE_SECURITY=false \
 #    mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME
 
 # Set up necessary directories and permissions
-RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles
+RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs  /pipeline/watchedFolders /pipeline/finishedFolders
 
 #    chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/fonts/opentype/noto /configs /customFiles
 
 # Copy necessary files
 COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh
 COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh
+COPY ./pipeline/ /pipeline/
 COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
 COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/
 COPY build/libs/*.jar app.jar

Dockerfile-ultra-lite

@@ -18,12 +18,12 @@ ENV DOCKER_ENABLE_SECURITY=false \
 
 # Set up necessary directories and permissions
 #RUN mkdir -p /scripts /configs /customFiles && \
-#    chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles
+#    chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs  /pipeline/watchedFolders /pipeline/finishedFolders
  
 RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles
 COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh
 COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh
-
+COPY ./pipeline/ /pipeline/
 COPY build/libs/*.jar app.jar
 
 # Set font cache and permissions

build.gradle

@@ -8,7 +8,7 @@ plugins {
 }
 
 group = 'stirling.software'
-version = '0.17.2'
+version = '0.18.0'
 sourceCompatibility = '17'
 
 repositories {

pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json

@@ -0,0 +1,39 @@
+{
+  "name": "Prepare-pdfs-for-email",
+  "pipeline": [
+    {
+      "operation": "/api/v1/misc/repair",
+      "parameters": {}
+    },
+    {
+      "operation": "/api/v1/security/sanitize-pdf",
+      "parameters": {
+        "removeJavaScript": true,
+        "removeEmbeddedFiles": false,
+        "removeMetadata": false,
+        "removeLinks": false,
+        "removeFonts": false
+      }
+    },
+    {
+      "operation": "/api/v1/misc/compress-pdf",
+      "parameters": {
+        "optimizeLevel": 2,
+        "expectedOutputSize": ""
+      }
+    },
+    {
+      "operation": "/api/v1/general/split-by-size-or-count",
+      "parameters": {
+        "splitType": 0,
+        "splitValue": "15MB"
+      }
+    }
+  ],
+  "_examples": {
+    "outputDir": "{outputFolder}/{folderName}",
+    "outputFileName": "{filename}-{pipelineName}-{date}-{time}"
+  },
+  "outputDir": "httpWebRequest",
+  "outputFileName": "{filename}"
+}

pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json

@@ -0,0 +1,30 @@
+{
+  "name": "split-rotate-auto-rename",
+  "pipeline": [
+    {
+      "operation": "/api/v1/general/split-pdf-by-sections",
+      "parameters": {
+        "horizontalDivisions": 2,
+        "verticalDivisions": 2
+      }
+    },
+    {
+      "operation": "/api/v1/general/rotate-pdf",
+      "parameters": {
+        "angle": 90
+      }
+    },
+    {
+      "operation": "/api/v1/misc/auto-rename",
+      "parameters": {
+        "useFirstTextAsFallback": false
+      }
+    }
+  ],
+  "_examples": {
+    "outputDir": "{outputFolder}/{folderName}",
+    "outputFileName": "{filename}-{pipelineName}-{date}-{time}"
+  },
+  "outputDir": "httpWebRequest",
+  "outputFileName": "{filename}"
+}

src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java

@@ -254,8 +254,10 @@ public class PipelineController {
 	List<Resource> processFiles(List<Resource> outputFiles, String jsonString) throws Exception {
 		
 		ObjectMapper mapper = new ObjectMapper();
-		JsonNode jsonNode = mapper.readTree(jsonString);
+		logger.info("Running jsonString {}", jsonString);
 		
+		JsonNode jsonNode = mapper.readTree(jsonString);
+		logger.info("Running jsonNode {}", jsonNode);
 		JsonNode pipelineNode = jsonNode.get("pipeline");
 		logger.info("Running pipelineNode: {}", pipelineNode);
 		ByteArrayOutputStream logStream = new ByteArrayOutputStream();
@@ -364,12 +366,7 @@ public class PipelineController {
 			
 		logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length());
 		
-		ObjectMapper mapper = new ObjectMapper();
-		JsonNode jsonNode = mapper.readTree(jsonString);
 	
-		JsonNode pipelineNode = jsonNode.get("pipeline");
-
-		boolean hasErrors = false;
 		List<Resource> outputFiles = new ArrayList<>();
 
 		for (File file : files) {

src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java

@@ -42,6 +42,8 @@ public class GeneralWebController {
 	    model.addAttribute("currentPage", "pipeline");
 
 	    List<String> pipelineConfigs = new ArrayList<>();
+	    List<Map<String, String>> pipelineConfigsWithNames = new ArrayList<>();
+	    
 	    if(new File("./pipeline/defaultWebUIConfigs/").exists()) {
 		    try (Stream<Path> paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) {
 		        List<Path> jsonFiles = paths
@@ -53,7 +55,7 @@ public class GeneralWebController {
 		            String content = Files.readString(jsonFile, StandardCharsets.UTF_8);
 		            pipelineConfigs.add(content);
 		        }
-		        List<Map<String, String>> pipelineConfigsWithNames = new ArrayList<>();
+		        
 		        for (String config : pipelineConfigs) {
 		        	Map<String, Object> jsonContent = new ObjectMapper().readValue(config, new TypeReference<Map<String, Object>>(){});
 	
@@ -63,12 +65,21 @@ public class GeneralWebController {
 		            configWithName.put("name", name);
 		            pipelineConfigsWithNames.add(configWithName);
 		        }
-		        model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames);
+		        
+		        
+		        
 	
 		    } catch (IOException e) {
 		        e.printStackTrace();
 		    }
 		}
+	    if(pipelineConfigsWithNames.size() == 0) {
+	    	 Map<String, String> configWithName = new HashMap<>();
+	            configWithName.put("json", "");
+	            configWithName.put("name", "No preloaded configs found");
+	            pipelineConfigsWithNames.add(configWithName);
+	    }
+	    model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames);
 
 	    model.addAttribute("pipelineConfigs", pipelineConfigs);
 

src/main/resources/static/js/pipeline.js

@@ -12,31 +12,17 @@ function validatePipeline() {
 		if (currentOperation === '/add-password') {
 			containsAddPassword = true;
 		}
-		console.log(currentOperation);
+
-		console.log(apiDocs[currentOperation]);
 		let currentOperationDescription = apiDocs[currentOperation]?.post?.description || "";
 		let nextOperationDescription = apiDocs[nextOperation]?.post?.description || "";
 
-		console.log("currentOperationDescription", currentOperationDescription);
-		console.log("nextOperationDescription", nextOperationDescription);
-
-		
 		// Strip off 'ZIP-' prefix
 		currentOperationDescription = currentOperationDescription.replace("ZIP-", '');
 		nextOperationDescription = nextOperationDescription.replace("ZIP-", '');
 		
-		console.log("currentOperationDescription", currentOperationDescription);
-		console.log("nextOperationDescription", nextOperationDescription);
-		
 		let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || "";
 		let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || "";
 
-
-		
-		
-		console.log("Operation " + currentOperation + " Output: " + currentOperationOutput);
-		console.log("Operation " + nextOperation + " Input: " + nextOperationInput);
-
 		// Splitting in case of multiple possible output/input
 		let currentOperationOutputArr = currentOperationOutput.split('/');
 		let nextOperationInputArr = nextOperationInput.split('/');
@@ -78,14 +64,14 @@ document.getElementById('submitConfigBtn').addEventListener('click', function()
 		return;
 	}
 	let selectedOperation = document.getElementById('operationsDropdown').value;
-	let parameters = operationSettings[selectedOperation] || {};
 
+	
+
+	var pipelineName = document.getElementById('pipelineName').value;
+	let pipelineList = document.getElementById('pipelineList').children;
 	let pipelineConfig = {
-		"name": "uniquePipelineName",
+		"name": pipelineName,
-		"pipeline": [{
+		"pipeline": [],
-			"operation": selectedOperation,
-			"parameters": parameters
-		}],
 		"_examples": {
 			"outputDir": "{outputFolder}/{folderName}",
 			"outputFileName": "{filename}-{pipelineName}-{date}-{time}"
@@ -94,6 +80,28 @@ document.getElementById('submitConfigBtn').addEventListener('click', function()
 		"outputFileName": "{filename}"
 	};
 
+	for (let i = 0; i < pipelineList.length; i++) {
+		let operationName = pipelineList[i].querySelector('.operationName').textContent;
+		let parameters = operationSettings[operationName] || {};
+
+		pipelineConfig.pipeline.push({
+			"operation": operationName,
+			"parameters": parameters
+		});
+	}
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+
 	let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2);
 
 	let formData = new FormData();
@@ -111,32 +119,35 @@ document.getElementById('submitConfigBtn').addEventListener('click', function()
 	console.log("formData", formData);
 
 	fetch('/api/v1/pipeline/handleData', {
-		method: 'POST',
+    method: 'POST',
-		body: formData
+    body: formData
 	})
-		.then(response => response.blob())
+	.then(response => {
-		.then(blob => {
+	    // Save the response to use it later
+	    const responseToUseLater = response;
 	
-			let url = window.URL.createObjectURL(blob);
+	    return response.blob().then(blob => {
-			let a = document.createElement('a');
+	        let url = window.URL.createObjectURL(blob);
-			a.href = url;
+	        let a = document.createElement('a');
+	        a.href = url;
 	
+	        // Use responseToUseLater instead of response
+	        const contentDisposition = responseToUseLater.headers.get('Content-Disposition');
+	        let filename = 'download';
+	        if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) {
+	            filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim();
+	        }
+	        a.download = filename;
 	        
-			const contentDisposition = response.headers.get('Content-Disposition');
+	        document.body.appendChild(a);
-			let filename = 'download';
+	        a.click();
-			if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) {
+	        a.remove();
-				filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim();
+	    });
-			}
+	})
-			a.download = filename;
+	.catch((error) => {
+	    console.error('Error:', error);
+	});
 
-			
-			document.body.appendChild(a);
-			a.click();
-			a.remove();
-		})
-		.catch((error) => {
-			console.error('Error:', error);
-		});
 });
 
 let apiDocs = {};
@@ -170,7 +181,6 @@ fetch('v1/api-docs')
 				operationsByTag[operationTag].push(operationPath);
 			}
 		});
-		console.log("operationsByTag", operationsByTag);
 		// Specify the order of tags
 		let tagOrder = ["General", "Security", "Convert", "Misc", "Filter"];
 
@@ -188,10 +198,7 @@ fetch('v1/api-docs')
             
 					
 					if(operationPath.includes("/convert")){
-						console.log("operationPathDisplay", operationPathDisplay);
 						operationPathDisplay = operationPathDisplay.replace(/^\//, '').replaceAll("/", " to ");
-
-						console.log("operationPathDisplay2", operationPathDisplay);
 					} else {
 						operationPathDisplay = operationPathDisplay.replace(/\//g, ''); // Remove slashes
 					}
@@ -223,7 +230,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
 	    } else if (postMethod.requestBody && postMethod.requestBody.content['multipart/form-data']) {
 	        // Extract the reference key
 	        const refKey = postMethod.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop();
-			console.log("refKey", refKey);
 	        // Check if the referenced schema exists and has properties
 	        if (apiSchemas[refKey] && Object.keys(apiSchemas[refKey].properties).length > 0) {
 	            hasSettings = true;
@@ -296,7 +302,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
 			// If the parameter name is 'fileInput', return early to skip the rest of this iteration
     		if (parameter.name === 'fileInput') return;
     
-    		console.log("parameter", parameter);
 			let parameterDiv = document.createElement('div');
 			parameterDiv.className = "mb-3";
 
@@ -407,7 +412,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
 			event.preventDefault();
 			let settings = {};
 			operationData.forEach(parameter => {
-				console.log("parameter.name", parameter.name);
 				if(parameter.name !== "fileInput"){
 					let value = document.getElementById(parameter.name).value;
 					switch (parameter.schema.type) {
@@ -432,7 +436,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
 				}
 			});
 			operationSettings[operation] = settings;
-			console.log(settings);
 			//pipelineSettingsModal.style.display = "none";
 		});
 		pipelineSettingsContent.appendChild(saveButton);

src/main/resources/templates/pipeline.html

@@ -36,7 +36,7 @@
 			<div class="container">
 				<div class="row justify-content-center">
 				
-				<h1>Pipeline Menu (Huge work in progress, very buggy!)</h1>
+				<h1>(Alpha) Pipeline Menu (Huge work in progress, very buggy!)</h1>
 						 <div class="bordered-box">
                         <div class="text-end text-top">
                          <button id="uploadPipelineBtn" class="btn btn-primary">Upload
@@ -48,7 +48,6 @@
                         <div class="center-element">
                             <div class="element-margin">
 							    <select id="pipelineSelect" class="custom-select">
-							        <option value="">Select a pipeline</option>
 							        <th:block th:each="config : ${pipelineConfigsWithNames}">
 							            <option th:value="${config.json}" th:text="${config.name}"></option>
 							        </th:block>
@@ -63,14 +62,18 @@
                         </div>
                     </div>
 
-					<h3>Current  Limitations</h3>
+					<h3>Current Limitations</h3>
-					<p>Cant have more than one of the same operation</p>
+					<ul>
-					<p>Cant input additional files via UI</p>
+					    <li>Cannot have more than one of the same operation</li>
+					    <li>Cannot input additional files via UI</li>
+					    <li>Does not work with multi-input functions yet (like merges)</li>
+					</ul>
 					
-					
+					<h3>How it Works Notes</h3>
-					<h3>How it works notes</h3>
+					<ul>
-					<p>Configre pipeline config file and input files to run files against it</p>
+					    <li>Configure the pipeline config file and input files to run files against it</li>
-					<p>For reuse, download config file and reupload it when needed or place in /pipeline/defaultWebUIConfigs/ to auto load in webUI for all users</p>
+					    <li>For reuse, download the config file and re-upload it when needed, or place it in /pipeline/defaultWebUIConfigs/ to auto-load in the web UI for all users</li>
+					</ul>