DGNum/Stirling-PDF
15
0
Fork 0
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity

pipeline

Browse source
This commit is contained in:
Anthony Stirling 2023-12-23 15:47:18 +00:00
parent 93f12d1313
commit 03450454c5
11 changed files with 164 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -15,8 +15,8 @@ local.properties
.classpath
.project
version.properties
pipeline/
pipeline/watchedFolders/
pipeline/finishedFolders/
#### Stirling-PDF Files ###
customFiles/
configs/

3
Dockerfile
View file

@ -18,13 +18,14 @@ ENV DOCKER_ENABLE_SECURITY=false \
## mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME
# Set up necessary directories and permissions
RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles
RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders
##&& \
## chown -R stirlingpdfuser:stirlingpdfgroup /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles && \
## chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/tesseract-ocr-original
# Copy necessary files
COPY ./scripts/* /scripts/
COPY ./pipeline/ /pipeline/
COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/
COPY build/libs/*.jar app.jar

3
Dockerfile-lite
View file

@ -28,13 +28,14 @@ ENV DOCKER_ENABLE_SECURITY=false \
# mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME
# Set up necessary directories and permissions
RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles
RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders
# chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/fonts/opentype/noto /configs /customFiles
# Copy necessary files
COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh
COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh
COPY ./pipeline/ /pipeline/
COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/
COPY build/libs/*.jar app.jar

4
Dockerfile-ultra-lite
View file

@ -18,12 +18,12 @@ ENV DOCKER_ENABLE_SECURITY=false \
# Set up necessary directories and permissions
#RUN mkdir -p /scripts /configs /customFiles && \
# chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles
# chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders
RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles
COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh
COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh
COPY ./pipeline/ /pipeline/
COPY build/libs/*.jar app.jar
# Set font cache and permissions

2
build.gradle
View file

@ -8,7 +8,7 @@ plugins {
}
group = 'stirling.software'
version = '0.17.2'
version = '0.18.0'
sourceCompatibility = '17'
repositories {

39
pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json Normal file
View file

@ -0,0 +1,39 @@
{
"name": "Prepare-pdfs-for-email",
"pipeline": [
{
"operation": "/api/v1/misc/repair",
"parameters": {}
},
{
"operation": "/api/v1/security/sanitize-pdf",
"parameters": {
"removeJavaScript": true,
"removeEmbeddedFiles": false,
"removeMetadata": false,
"removeLinks": false,
"removeFonts": false
}
},
{
"operation": "/api/v1/misc/compress-pdf",
"parameters": {
"optimizeLevel": 2,
"expectedOutputSize": ""
}
},
{
"operation": "/api/v1/general/split-by-size-or-count",
"parameters": {
"splitType": 0,
"splitValue": "15MB"
}
}
],
"_examples": {
"outputDir": "{outputFolder}/{folderName}",
"outputFileName": "{filename}-{pipelineName}-{date}-{time}"
},
"outputDir": "httpWebRequest",
"outputFileName": "{filename}"
}

30
pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json Normal file
View file

@ -0,0 +1,30 @@
{
"name": "split-rotate-auto-rename",
"pipeline": [
{
"operation": "/api/v1/general/split-pdf-by-sections",
"parameters": {
"horizontalDivisions": 2,
"verticalDivisions": 2
}
},
{
"operation": "/api/v1/general/rotate-pdf",
"parameters": {
"angle": 90
}
},
{
"operation": "/api/v1/misc/auto-rename",
"parameters": {
"useFirstTextAsFallback": false
}
}
],
"_examples": {
"outputDir": "{outputFolder}/{folderName}",
"outputFileName": "{filename}-{pipelineName}-{date}-{time}"
},
"outputDir": "httpWebRequest",
"outputFileName": "{filename}"
}

9
src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java
View file

@ -254,8 +254,10 @@ public class PipelineController {
List<Resource> processFiles(List<Resource> outputFiles, String jsonString) throws Exception {
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(jsonString);
logger.info("Running jsonString {}", jsonString);
JsonNode jsonNode = mapper.readTree(jsonString);
logger.info("Running jsonNode {}", jsonNode);
JsonNode pipelineNode = jsonNode.get("pipeline");
logger.info("Running pipelineNode: {}", pipelineNode);
ByteArrayOutputStream logStream = new ByteArrayOutputStream();
@ -364,12 +366,7 @@ public class PipelineController {
logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length());
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(jsonString);
JsonNode pipelineNode = jsonNode.get("pipeline");
boolean hasErrors = false;
List<Resource> outputFiles = new ArrayList<>();
for (File file : files) {

15
src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java
View file

@ -42,6 +42,8 @@ public class GeneralWebController {
model.addAttribute("currentPage", "pipeline");
List<String> pipelineConfigs = new ArrayList<>();
List<Map<String, String>> pipelineConfigsWithNames = new ArrayList<>();
if(new File("./pipeline/defaultWebUIConfigs/").exists()) {
try (Stream<Path> paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) {
List<Path> jsonFiles = paths
@ -53,7 +55,7 @@ public class GeneralWebController {
String content = Files.readString(jsonFile, StandardCharsets.UTF_8);
pipelineConfigs.add(content);
}
List<Map<String, String>> pipelineConfigsWithNames = new ArrayList<>();
for (String config : pipelineConfigs) {
Map<String, Object> jsonContent = new ObjectMapper().readValue(config, new TypeReference<Map<String, Object>>(){});
@ -63,12 +65,21 @@ public class GeneralWebController {
configWithName.put("name", name);
pipelineConfigsWithNames.add(configWithName);
}
model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames);
} catch (IOException e) {
e.printStackTrace();
}
}
if(pipelineConfigsWithNames.size() == 0) {
Map<String, String> configWithName = new HashMap<>();
configWithName.put("json", "");
configWithName.put("name", "No preloaded configs found");
pipelineConfigsWithNames.add(configWithName);
}
model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames);
model.addAttribute("pipelineConfigs", pipelineConfigs);

103
src/main/resources/static/js/pipeline.js
View file

@ -12,31 +12,17 @@ function validatePipeline() {
if (currentOperation === '/add-password') {
containsAddPassword = true;
}
console.log(currentOperation);
console.log(apiDocs[currentOperation]);
let currentOperationDescription = apiDocs[currentOperation]?.post?.description || "";
let nextOperationDescription = apiDocs[nextOperation]?.post?.description || "";
console.log("currentOperationDescription", currentOperationDescription);
console.log("nextOperationDescription", nextOperationDescription);
// Strip off 'ZIP-' prefix
currentOperationDescription = currentOperationDescription.replace("ZIP-", '');
nextOperationDescription = nextOperationDescription.replace("ZIP-", '');
console.log("currentOperationDescription", currentOperationDescription);
console.log("nextOperationDescription", nextOperationDescription);
let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || "";
let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || "";
console.log("Operation " + currentOperation + " Output: " + currentOperationOutput);
console.log("Operation " + nextOperation + " Input: " + nextOperationInput);
// Splitting in case of multiple possible output/input
let currentOperationOutputArr = currentOperationOutput.split('/');
let nextOperationInputArr = nextOperationInput.split('/');
@ -78,14 +64,14 @@ document.getElementById('submitConfigBtn').addEventListener('click', function()
return;
}
let selectedOperation = document.getElementById('operationsDropdown').value;
let parameters = operationSettings[selectedOperation] || {};
var pipelineName = document.getElementById('pipelineName').value;
let pipelineList = document.getElementById('pipelineList').children;
let pipelineConfig = {
"name": "uniquePipelineName",
"pipeline": [{
"operation": selectedOperation,
"parameters": parameters
}],
"name": pipelineName,
"pipeline": [],
"_examples": {
"outputDir": "{outputFolder}/{folderName}",
"outputFileName": "{filename}-{pipelineName}-{date}-{time}"
@ -94,6 +80,28 @@ document.getElementById('submitConfigBtn').addEventListener('click', function()
"outputFileName": "{filename}"
};
for (let i = 0; i < pipelineList.length; i++) {
let operationName = pipelineList[i].querySelector('.operationName').textContent;
let parameters = operationSettings[operationName] || {};
pipelineConfig.pipeline.push({
"operation": operationName,
"parameters": parameters
});
}
let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2);
let formData = new FormData();
@ -111,32 +119,35 @@ document.getElementById('submitConfigBtn').addEventListener('click', function()
console.log("formData", formData);
fetch('/api/v1/pipeline/handleData', {
method: 'POST',
body: formData
method: 'POST',
body: formData
})
.then(response => response.blob())
.then(blob => {
.then(response => {
// Save the response to use it later
const responseToUseLater = response;
let url = window.URL.createObjectURL(blob);
let a = document.createElement('a');
a.href = url;
return response.blob().then(blob => {
let url = window.URL.createObjectURL(blob);
let a = document.createElement('a');
a.href = url;
// Use responseToUseLater instead of response
const contentDisposition = responseToUseLater.headers.get('Content-Disposition');
let filename = 'download';
if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) {
filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim();
}
a.download = filename;
const contentDisposition = response.headers.get('Content-Disposition');
let filename = 'download';
if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) {
filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim();
}
a.download = filename;
document.body.appendChild(a);
a.click();
a.remove();
});
})
.catch((error) => {
console.error('Error:', error);
});
document.body.appendChild(a);
a.click();
a.remove();
})
.catch((error) => {
console.error('Error:', error);
});
});
let apiDocs = {};
@ -170,7 +181,6 @@ fetch('v1/api-docs')
operationsByTag[operationTag].push(operationPath);
}
});
console.log("operationsByTag", operationsByTag);
// Specify the order of tags
let tagOrder = ["General", "Security", "Convert", "Misc", "Filter"];
@ -188,10 +198,7 @@ fetch('v1/api-docs')
if(operationPath.includes("/convert")){
console.log("operationPathDisplay", operationPathDisplay);
operationPathDisplay = operationPathDisplay.replace(/^\//, '').replaceAll("/", " to ");
console.log("operationPathDisplay2", operationPathDisplay);
} else {
operationPathDisplay = operationPathDisplay.replace(/\//g, ''); // Remove slashes
}
@ -223,7 +230,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
} else if (postMethod.requestBody && postMethod.requestBody.content['multipart/form-data']) {
// Extract the reference key
const refKey = postMethod.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop();
console.log("refKey", refKey);
// Check if the referenced schema exists and has properties
if (apiSchemas[refKey] && Object.keys(apiSchemas[refKey].properties).length > 0) {
hasSettings = true;
@ -296,7 +302,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
// If the parameter name is 'fileInput', return early to skip the rest of this iteration
if (parameter.name === 'fileInput') return;
console.log("parameter", parameter);
let parameterDiv = document.createElement('div');
parameterDiv.className = "mb-3";
@ -407,7 +412,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
event.preventDefault();
let settings = {};
operationData.forEach(parameter => {
console.log("parameter.name", parameter.name);
if(parameter.name !== "fileInput"){
let value = document.getElementById(parameter.name).value;
switch (parameter.schema.type) {
@ -432,7 +436,6 @@ document.getElementById('addOperationBtn').addEventListener('click', function()
}
});
operationSettings[operation] = settings;
console.log(settings);
//pipelineSettingsModal.style.display = "none";
});
pipelineSettingsContent.appendChild(saveButton);

21
src/main/resources/templates/pipeline.html
View file

@ -36,7 +36,7 @@
<div class="container">
<div class="row justify-content-center">
<h1>Pipeline Menu (Huge work in progress, very buggy!)</h1>
<h1>(Alpha) Pipeline Menu (Huge work in progress, very buggy!)</h1>
<div class="bordered-box">
<div class="text-end text-top">
<button id="uploadPipelineBtn" class="btn btn-primary">Upload
@ -48,7 +48,6 @@
<div class="center-element">
<div class="element-margin">
<select id="pipelineSelect" class="custom-select">
<option value="">Select a pipeline</option>
<th:block th:each="config : ${pipelineConfigsWithNames}">
<option th:value="${config.json}" th:text="${config.name}"></option>
</th:block>
@ -63,14 +62,18 @@
</div>
</div>
<h3>Current Limitations</h3>
<p>Cant have more than one of the same operation</p>
<p>Cant input additional files via UI</p>
<h3>Current Limitations</h3>
<ul>
<li>Cannot have more than one of the same operation</li>
<li>Cannot input additional files via UI</li>
<li>Does not work with multi-input functions yet (like merges)</li>
</ul>
<h3>How it works notes</h3>
<p>Configre pipeline config file and input files to run files against it</p>
<p>For reuse, download config file and reupload it when needed or place in /pipeline/defaultWebUIConfigs/ to auto load in webUI for all users</p>
<h3>How it Works Notes</h3>
<ul>
<li>Configure the pipeline config file and input files to run files against it</li>
<li>For reuse, download the config file and re-upload it when needed, or place it in /pipeline/defaultWebUIConfigs/ to auto-load in the web UI for all users</li>
</ul>