diff --git a/.gitignore b/.gitignore index 14f1c139..a4e70f57 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,8 @@ local.properties .classpath .project version.properties -pipeline/ - +pipeline/watchedFolders/ +pipeline/finishedFolders/ #### Stirling-PDF Files ### customFiles/ configs/ diff --git a/Dockerfile b/Dockerfile index 6d830346..9b1f141a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,13 +18,14 @@ ENV DOCKER_ENABLE_SECURITY=false \ ## mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME # Set up necessary directories and permissions -RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles +RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders ##&& \ ## chown -R stirlingpdfuser:stirlingpdfgroup /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles && \ ## chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/tesseract-ocr-original # Copy necessary files COPY ./scripts/* /scripts/ +COPY ./pipeline/ /pipeline/ COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/ COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/ COPY build/libs/*.jar app.jar diff --git a/Dockerfile-lite b/Dockerfile-lite index 9d734942..2e60caab 100644 --- a/Dockerfile-lite +++ b/Dockerfile-lite @@ -28,13 +28,14 @@ ENV DOCKER_ENABLE_SECURITY=false \ # mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME # Set up necessary directories and permissions -RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles +RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders # chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/fonts/opentype/noto /configs /customFiles # Copy necessary files COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh +COPY ./pipeline/ /pipeline/ COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/ COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/ COPY build/libs/*.jar app.jar diff --git a/Dockerfile-ultra-lite b/Dockerfile-ultra-lite index b49b9023..ea7dffe5 100644 --- a/Dockerfile-ultra-lite +++ b/Dockerfile-ultra-lite @@ -18,12 +18,12 @@ ENV DOCKER_ENABLE_SECURITY=false \ # Set up necessary directories and permissions #RUN mkdir -p /scripts /configs /customFiles && \ -# chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles - +# chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders + RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh - +COPY ./pipeline/ /pipeline/ COPY build/libs/*.jar app.jar # Set font cache and permissions diff --git a/build.gradle b/build.gradle index f3efe33a..1c3592a0 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ plugins { } group = 'stirling.software' -version = '0.17.2' +version = '0.18.0' sourceCompatibility = '17' repositories { diff --git a/pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json b/pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json new file mode 100644 index 00000000..ba3228dc --- /dev/null +++ b/pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json @@ -0,0 +1,39 @@ +{ + "name": "Prepare-pdfs-for-email", + "pipeline": [ + { + "operation": "/api/v1/misc/repair", + "parameters": {} + }, + { + "operation": "/api/v1/security/sanitize-pdf", + "parameters": { + "removeJavaScript": true, + "removeEmbeddedFiles": false, + "removeMetadata": false, + "removeLinks": false, + "removeFonts": false + } + }, + { + "operation": "/api/v1/misc/compress-pdf", + "parameters": { + "optimizeLevel": 2, + "expectedOutputSize": "" + } + }, + { + "operation": "/api/v1/general/split-by-size-or-count", + "parameters": { + "splitType": 0, + "splitValue": "15MB" + } + } + ], + "_examples": { + "outputDir": "{outputFolder}/{folderName}", + "outputFileName": "{filename}-{pipelineName}-{date}-{time}" + }, + "outputDir": "httpWebRequest", + "outputFileName": "{filename}" +} \ No newline at end of file diff --git a/pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json b/pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json new file mode 100644 index 00000000..3e8dea7f --- /dev/null +++ b/pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json @@ -0,0 +1,30 @@ +{ + "name": "split-rotate-auto-rename", + "pipeline": [ + { + "operation": "/api/v1/general/split-pdf-by-sections", + "parameters": { + "horizontalDivisions": 2, + "verticalDivisions": 2 + } + }, + { + "operation": "/api/v1/general/rotate-pdf", + "parameters": { + "angle": 90 + } + }, + { + "operation": "/api/v1/misc/auto-rename", + "parameters": { + "useFirstTextAsFallback": false + } + } + ], + "_examples": { + "outputDir": "{outputFolder}/{folderName}", + "outputFileName": "{filename}-{pipelineName}-{date}-{time}" + }, + "outputDir": "httpWebRequest", + "outputFileName": "{filename}" +} \ No newline at end of file diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java index 568d4430..71df9f62 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java @@ -254,8 +254,10 @@ public class PipelineController { List processFiles(List outputFiles, String jsonString) throws Exception { ObjectMapper mapper = new ObjectMapper(); + logger.info("Running jsonString {}", jsonString); + JsonNode jsonNode = mapper.readTree(jsonString); - + logger.info("Running jsonNode {}", jsonNode); JsonNode pipelineNode = jsonNode.get("pipeline"); logger.info("Running pipelineNode: {}", pipelineNode); ByteArrayOutputStream logStream = new ByteArrayOutputStream(); @@ -364,12 +366,7 @@ public class PipelineController { logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - - JsonNode pipelineNode = jsonNode.get("pipeline"); - - boolean hasErrors = false; + List outputFiles = new ArrayList<>(); for (File file : files) { diff --git a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java index 411014a8..b670a129 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java @@ -42,6 +42,8 @@ public class GeneralWebController { model.addAttribute("currentPage", "pipeline"); List pipelineConfigs = new ArrayList<>(); + List> pipelineConfigsWithNames = new ArrayList<>(); + if(new File("./pipeline/defaultWebUIConfigs/").exists()) { try (Stream paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) { List jsonFiles = paths @@ -53,7 +55,7 @@ public class GeneralWebController { String content = Files.readString(jsonFile, StandardCharsets.UTF_8); pipelineConfigs.add(content); } - List> pipelineConfigsWithNames = new ArrayList<>(); + for (String config : pipelineConfigs) { Map jsonContent = new ObjectMapper().readValue(config, new TypeReference>(){}); @@ -63,12 +65,21 @@ public class GeneralWebController { configWithName.put("name", name); pipelineConfigsWithNames.add(configWithName); } - model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames); + + + } catch (IOException e) { e.printStackTrace(); } } + if(pipelineConfigsWithNames.size() == 0) { + Map configWithName = new HashMap<>(); + configWithName.put("json", ""); + configWithName.put("name", "No preloaded configs found"); + pipelineConfigsWithNames.add(configWithName); + } + model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames); model.addAttribute("pipelineConfigs", pipelineConfigs); diff --git a/src/main/resources/static/js/pipeline.js b/src/main/resources/static/js/pipeline.js index fa26f8f3..4f104a2a 100644 --- a/src/main/resources/static/js/pipeline.js +++ b/src/main/resources/static/js/pipeline.js @@ -12,31 +12,17 @@ function validatePipeline() { if (currentOperation === '/add-password') { containsAddPassword = true; } - console.log(currentOperation); - console.log(apiDocs[currentOperation]); + let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; - console.log("currentOperationDescription", currentOperationDescription); - console.log("nextOperationDescription", nextOperationDescription); - - // Strip off 'ZIP-' prefix currentOperationDescription = currentOperationDescription.replace("ZIP-", ''); nextOperationDescription = nextOperationDescription.replace("ZIP-", ''); - - console.log("currentOperationDescription", currentOperationDescription); - console.log("nextOperationDescription", nextOperationDescription); let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; - - - - console.log("Operation " + currentOperation + " Output: " + currentOperationOutput); - console.log("Operation " + nextOperation + " Input: " + nextOperationInput); - // Splitting in case of multiple possible output/input let currentOperationOutputArr = currentOperationOutput.split('/'); let nextOperationInputArr = nextOperationInput.split('/'); @@ -78,14 +64,14 @@ document.getElementById('submitConfigBtn').addEventListener('click', function() return; } let selectedOperation = document.getElementById('operationsDropdown').value; - let parameters = operationSettings[selectedOperation] || {}; + + + var pipelineName = document.getElementById('pipelineName').value; + let pipelineList = document.getElementById('pipelineList').children; let pipelineConfig = { - "name": "uniquePipelineName", - "pipeline": [{ - "operation": selectedOperation, - "parameters": parameters - }], + "name": pipelineName, + "pipeline": [], "_examples": { "outputDir": "{outputFolder}/{folderName}", "outputFileName": "{filename}-{pipelineName}-{date}-{time}" @@ -94,6 +80,28 @@ document.getElementById('submitConfigBtn').addEventListener('click', function() "outputFileName": "{filename}" }; + for (let i = 0; i < pipelineList.length; i++) { + let operationName = pipelineList[i].querySelector('.operationName').textContent; + let parameters = operationSettings[operationName] || {}; + + pipelineConfig.pipeline.push({ + "operation": operationName, + "parameters": parameters + }); + } + + + + + + + + + + + + + let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2); let formData = new FormData(); @@ -111,32 +119,35 @@ document.getElementById('submitConfigBtn').addEventListener('click', function() console.log("formData", formData); fetch('/api/v1/pipeline/handleData', { - method: 'POST', - body: formData + method: 'POST', + body: formData }) - .then(response => response.blob()) - .then(blob => { + .then(response => { + // Save the response to use it later + const responseToUseLater = response; + + return response.blob().then(blob => { + let url = window.URL.createObjectURL(blob); + let a = document.createElement('a'); + a.href = url; + + // Use responseToUseLater instead of response + const contentDisposition = responseToUseLater.headers.get('Content-Disposition'); + let filename = 'download'; + if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) { + filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim(); + } + a.download = filename; + + document.body.appendChild(a); + a.click(); + a.remove(); + }); + }) + .catch((error) => { + console.error('Error:', error); + }); - let url = window.URL.createObjectURL(blob); - let a = document.createElement('a'); - a.href = url; - - - const contentDisposition = response.headers.get('Content-Disposition'); - let filename = 'download'; - if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) { - filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim(); - } - a.download = filename; - - - document.body.appendChild(a); - a.click(); - a.remove(); - }) - .catch((error) => { - console.error('Error:', error); - }); }); let apiDocs = {}; @@ -170,7 +181,6 @@ fetch('v1/api-docs') operationsByTag[operationTag].push(operationPath); } }); - console.log("operationsByTag", operationsByTag); // Specify the order of tags let tagOrder = ["General", "Security", "Convert", "Misc", "Filter"]; @@ -188,10 +198,7 @@ fetch('v1/api-docs') if(operationPath.includes("/convert")){ - console.log("operationPathDisplay", operationPathDisplay); operationPathDisplay = operationPathDisplay.replace(/^\//, '').replaceAll("/", " to "); - - console.log("operationPathDisplay2", operationPathDisplay); } else { operationPathDisplay = operationPathDisplay.replace(/\//g, ''); // Remove slashes } @@ -223,7 +230,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function() } else if (postMethod.requestBody && postMethod.requestBody.content['multipart/form-data']) { // Extract the reference key const refKey = postMethod.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); - console.log("refKey", refKey); // Check if the referenced schema exists and has properties if (apiSchemas[refKey] && Object.keys(apiSchemas[refKey].properties).length > 0) { hasSettings = true; @@ -296,7 +302,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function() // If the parameter name is 'fileInput', return early to skip the rest of this iteration if (parameter.name === 'fileInput') return; - console.log("parameter", parameter); let parameterDiv = document.createElement('div'); parameterDiv.className = "mb-3"; @@ -407,7 +412,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function() event.preventDefault(); let settings = {}; operationData.forEach(parameter => { - console.log("parameter.name", parameter.name); if(parameter.name !== "fileInput"){ let value = document.getElementById(parameter.name).value; switch (parameter.schema.type) { @@ -432,7 +436,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function() } }); operationSettings[operation] = settings; - console.log(settings); //pipelineSettingsModal.style.display = "none"; }); pipelineSettingsContent.appendChild(saveButton); diff --git a/src/main/resources/templates/pipeline.html b/src/main/resources/templates/pipeline.html index 3b313f32..bb68cc3b 100644 --- a/src/main/resources/templates/pipeline.html +++ b/src/main/resources/templates/pipeline.html @@ -36,7 +36,7 @@
-

Pipeline Menu (Huge work in progress, very buggy!)

+

(Alpha) Pipeline Menu (Huge work in progress, very buggy!)