From 6ca9001fe6115ded1d147051795d870ae1c9c9bf Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 30 Dec 2023 13:42:24 +0000 Subject: [PATCH 01/23] enable status check without apikey --- .../software/SPDF/config/security/SecurityConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java index e0b439db..71394b09 100644 --- a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java +++ b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java @@ -90,7 +90,8 @@ public class SecurityConfiguration { return trimmedUri.startsWith("/login") || trimmedUri.endsWith(".svg") || trimmedUri.startsWith("/register") || trimmedUri.startsWith("/error") || trimmedUri.startsWith("/images/") || trimmedUri.startsWith("/public/") || - trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/"); + trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/") || + trimmedUri.startsWith("api/v1/info/status"); } ).permitAll() .anyRequest().authenticated() From c853465d1d356c96471a0139a38dd7801be79935 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 30 Dec 2023 18:56:07 +0000 Subject: [PATCH 02/23] tests --- .../docker-compose-latest-lite-security.yml | 31 +++++ .../docker-compose-latest-lite.yml | 30 +++++ .../docker-compose-latest-security.yml | 31 +++++ ...ker-compose-latest-ultra-lite-security.yml | 31 +++++ .../docker-compose-latest-ultra-lite.yml | 30 +++++ exampleYmlFiles/docker-compose-latest.yml | 31 +++++ .../security/SecurityConfiguration.java | 2 +- .../security/UserAuthenticationFilter.java | 1 + .../software/SPDF/utils/RequestUriUtils.java | 3 +- test.sh | 122 ++++++++++++++++++ 10 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 exampleYmlFiles/docker-compose-latest-lite-security.yml create mode 100644 exampleYmlFiles/docker-compose-latest-lite.yml create mode 100644 exampleYmlFiles/docker-compose-latest-security.yml create mode 100644 exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml create mode 100644 exampleYmlFiles/docker-compose-latest-ultra-lite.yml create mode 100644 exampleYmlFiles/docker-compose-latest.yml create mode 100644 test.sh diff --git a/exampleYmlFiles/docker-compose-latest-lite-security.yml b/exampleYmlFiles/docker-compose-latest-lite-security.yml new file mode 100644 index 00000000..98b95058 --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-lite-security.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Lite-Security + image: frooodle/s-pdf:latest-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] + interval: 30s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "true" + SECURITY_ENABLELOGIN: "true" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-lite.yml b/exampleYmlFiles/docker-compose-latest-lite.yml new file mode 100644 index 00000000..c2f18efe --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-lite.yml @@ -0,0 +1,30 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Lite + image: frooodle/s-pdf:latest-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] + interval: 20s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "false" + SECURITY_ENABLELOGIN: "false" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-security.yml b/exampleYmlFiles/docker-compose-latest-security.yml new file mode 100644 index 00000000..b36a08a3 --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-security.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Security + image: frooodle/s-pdf:latest-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] + interval: 20s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "true" + SECURITY_ENABLELOGIN: "true" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml new file mode 100644 index 00000000..7b0d6c43 --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Ultra-Lite-Security + image: frooodle/s-pdf:latest-ultra-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] + interval: 30s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "true" + SECURITY_ENABLELOGIN: "true" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml new file mode 100644 index 00000000..5848873d --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml @@ -0,0 +1,30 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Ultra-Lite + image: frooodle/s-pdf:latest-ultra-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] + interval: 20s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "false" + SECURITY_ENABLELOGIN: "false" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest.yml b/exampleYmlFiles/docker-compose-latest.yml new file mode 100644 index 00000000..e319695f --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF + image: frooodle/s-pdf:latest-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] + interval: 30s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "false" + SECURITY_ENABLELOGIN: "false" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java index 71394b09..18fad46b 100644 --- a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java +++ b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java @@ -91,7 +91,7 @@ public class SecurityConfiguration { trimmedUri.startsWith("/register") || trimmedUri.startsWith("/error") || trimmedUri.startsWith("/images/") || trimmedUri.startsWith("/public/") || trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/") || - trimmedUri.startsWith("api/v1/info/status"); + trimmedUri.startsWith("/api/v1/info/status"); } ).permitAll() .anyRequest().authenticated() diff --git a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java index ce77e5a4..2b98a9e5 100644 --- a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java +++ b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java @@ -102,6 +102,7 @@ public class UserAuthenticationFilter extends OncePerRequestFilter { contextPath + "/css/", contextPath + "/js/", contextPath + "/pdfjs/", + contextPath + "/api/v1/info/status", contextPath + "/site.webmanifest" }; diff --git a/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java index 0046ee9f..6f07b573 100644 --- a/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java +++ b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java @@ -9,7 +9,8 @@ public class RequestUriUtils { || requestURI.startsWith("/images/") || requestURI.startsWith("/public/") || requestURI.startsWith("/pdfjs/") - || requestURI.endsWith(".svg"); + || requestURI.endsWith(".svg") + || requestURI.startsWith("/api/v1/info/status"); } diff --git a/test.sh b/test.sh new file mode 100644 index 00000000..3b84ef42 --- /dev/null +++ b/test.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Function to check the health of the service with a timeout of 80 seconds +check_health() { + local service_name=$1 + local compose_file=$2 + local end=$((SECONDS+60)) + + echo "Waiting for $service_name to become healthy..." + until [ "$(docker inspect --format='{{json .State.Health.Status}}' "$service_name")" == '"healthy"' ] || [ $SECONDS -ge $end ]; do + sleep 10 + echo "Waiting..." + if [ $SECONDS -ge $end ]; then + echo "$service_name health check timed out after 80 seconds." + return 1 + fi + done + echo "$service_name is healthy!" + return 0 +} + +# Function to test a Docker Compose configuration +test_compose() { + local compose_file=$1 + local service_name=$2 + local status=0 + + echo "Testing $compose_file configuration..." + + # Start up the Docker Compose service + docker-compose -f "$compose_file" up -d + + # Wait for the service to become healthy + if check_health "$service_name" "$compose_file"; then + echo "$service_name test passed." + else + echo "$service_name test failed." + status=1 + fi + + # Perform additional tests if needed + + # Tear down the service + docker-compose -f "$compose_file" down + + return $status +} + +# Keep track of which tests passed and failed +declare -a passed_tests +declare -a failed_tests + +run_tests() { + local test_name=$1 + local compose_file=$2 + + if test_compose "$compose_file" "$test_name"; then + passed_tests+=("$test_name") + else + failed_tests+=("$test_name") + fi +} + +# Main testing routine +main() { + export DOCKER_ENABLE_SECURITY=false + ./gradlew clean build + + # Building Docker images + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-local -f ./Dockerfile . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-lite-local -f ./Dockerfile-lite . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-ultra-lite-local -f ./Dockerfile-ultra-lite . + + # Test each configuration + run_tests "Stirling-PDF-Ultra-Lite" "./exampleYmlFiles/docker-compose-latest-ultra-lite.yml" + run_tests "Stirling-PDF-Lite" "./exampleYmlFiles/docker-compose-latest-lite.yml" + run_tests "Stirling-PDF" "./exampleYmlFiles/docker-compose-latest.yml" + + export DOCKER_ENABLE_SECURITY=true + ./gradlew clean build + + # Building Docker images with security enabled + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-local -f ./Dockerfile . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-lite-local -f ./Dockerfile-lite . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-ultra-lite-local -f ./Dockerfile-ultra-lite . + + # Test each configuration with security + run_tests "Stirling-PDF-Ultra-Lite-Security" "./exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml" + run_tests "Stirling-PDF-Lite-Security" "./exampleYmlFiles/docker-compose-latest-lite-security.yml" + run_tests "Stirling-PDF-Security" "./exampleYmlFiles/docker-compose-latest-security.yml" + + # Report results + echo "All tests completed." + + + + if [ ${#passed_tests[@]} -ne 0 ]; then + echo "Passed tests:" + fi + for test in "${passed_tests[@]}"; do + echo -e "\e[32m$test\e[0m" # Green color for passed tests + done + + if [ ${#failed_tests[@]} -ne 0 ]; then + echo "Failed tests:" + fi + for test in "${failed_tests[@]}"; do + echo -e "\e[31m$test\e[0m" # Red color for failed tests + done + + # Check if there are any failed tests and exit with an error code if so + if [ ${#failed_tests[@]} -ne 0 ]; then + echo "Some tests failed." + exit 1 + else + echo "All tests passed successfully." + exit 0 + fi + +} + +main From eda91cc55610ed9731351e2e2a6a1c42f0199bab Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 30 Dec 2023 21:32:04 +0000 Subject: [PATCH 03/23] tests --- .../docker-compose-latest-lite-security.yml | 4 +-- .../docker-compose-latest-lite.yml | 4 +-- .../docker-compose-latest-security.yml | 4 +-- ...ker-compose-latest-ultra-lite-security.yml | 4 +-- .../docker-compose-latest-ultra-lite.yml | 4 +-- exampleYmlFiles/docker-compose-latest.yml | 4 +-- test.sh | 28 +++++++++++-------- 7 files changed, 28 insertions(+), 24 deletions(-) diff --git a/exampleYmlFiles/docker-compose-latest-lite-security.yml b/exampleYmlFiles/docker-compose-latest-lite-security.yml index 98b95058..cb82fdb0 100644 --- a/exampleYmlFiles/docker-compose-latest-lite-security.yml +++ b/exampleYmlFiles/docker-compose-latest-lite-security.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] - interval: 30s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-lite.yml b/exampleYmlFiles/docker-compose-latest-lite.yml index c2f18efe..920ae8d8 100644 --- a/exampleYmlFiles/docker-compose-latest-lite.yml +++ b/exampleYmlFiles/docker-compose-latest-lite.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] - interval: 20s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-security.yml b/exampleYmlFiles/docker-compose-latest-security.yml index b36a08a3..a480c27d 100644 --- a/exampleYmlFiles/docker-compose-latest-security.yml +++ b/exampleYmlFiles/docker-compose-latest-security.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] - interval: 20s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml index 7b0d6c43..a8de634f 100644 --- a/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] - interval: 30s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml index 5848873d..b3ad7b00 100644 --- a/exampleYmlFiles/docker-compose-latest-ultra-lite.yml +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] - interval: 20s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest.yml b/exampleYmlFiles/docker-compose-latest.yml index e319695f..21fff579 100644 --- a/exampleYmlFiles/docker-compose-latest.yml +++ b/exampleYmlFiles/docker-compose-latest.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] - interval: 30s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/test.sh b/test.sh index 3b84ef42..9617ba14 100644 --- a/test.sh +++ b/test.sh @@ -6,16 +6,17 @@ check_health() { local compose_file=$2 local end=$((SECONDS+60)) - echo "Waiting for $service_name to become healthy..." - until [ "$(docker inspect --format='{{json .State.Health.Status}}' "$service_name")" == '"healthy"' ] || [ $SECONDS -ge $end ]; do - sleep 10 - echo "Waiting..." - if [ $SECONDS -ge $end ]; then - echo "$service_name health check timed out after 80 seconds." - return 1 - fi - done - echo "$service_name is healthy!" + echo -n "Waiting for $service_name to become healthy..." + until [ "$(docker inspect --format='{{json .State.Health.Status}}' "$service_name")" == '"healthy"' ] || [ $SECONDS -ge $end ]; do + sleep 3 + echo -n "." + if [ $SECONDS -ge $end ]; then + echo -e "\n$service_name health check timed out after 80 seconds." + return 1 + fi + done + echo -e "\n$service_name is healthy!" + return 0 } @@ -63,6 +64,8 @@ run_tests() { # Main testing routine main() { + SECONDS=0 + export DOCKER_ENABLE_SECURITY=false ./gradlew clean build @@ -90,8 +93,7 @@ main() { run_tests "Stirling-PDF-Security" "./exampleYmlFiles/docker-compose-latest-security.yml" # Report results - echo "All tests completed." - + echo "All tests completed in $SECONDS seconds." if [ ${#passed_tests[@]} -ne 0 ]; then @@ -108,6 +110,8 @@ main() { echo -e "\e[31m$test\e[0m" # Red color for failed tests done + + # Check if there are any failed tests and exit with an error code if so if [ ${#failed_tests[@]} -ne 0 ]; then echo "Some tests failed." From f535387ac4dc943eb0675a1ca02849b243e9fe6b Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 31 Dec 2023 13:05:38 +0000 Subject: [PATCH 04/23] pipeline enhance for MI --- .../api/pipeline/PipelineProcessor.java | 4 +- src/main/resources/static/js/pipeline.js | 997 +++++++++--------- src/main/resources/templates/pipeline.html | 2 + 3 files changed, 521 insertions(+), 482 deletions(-) diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java index 8b4b2ef4..c2841a05 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java @@ -138,7 +138,7 @@ public class PipelineProcessor { hasErrors = true; } - outputFiles = newOutputFiles; + } } else { @@ -177,11 +177,13 @@ public class PipelineProcessor { } } logPrintStream.close(); + outputFiles = newOutputFiles; } if (hasErrors) { logger.error("Errors occurred during processing. Log: {}", logStream.toString()); } + return outputFiles; } diff --git a/src/main/resources/static/js/pipeline.js b/src/main/resources/static/js/pipeline.js index 4fcde3a0..8db09c48 100644 --- a/src/main/resources/static/js/pipeline.js +++ b/src/main/resources/static/js/pipeline.js @@ -1,490 +1,511 @@ -document.getElementById('validateButton').addEventListener('click', function(event) { - event.preventDefault(); - validatePipeline(); -}); -function validatePipeline() { - let pipelineListItems = document.getElementById('pipelineList').children; - let isValid = true; - let containsAddPassword = false; - for (let i = 0; i < pipelineListItems.length - 1; i++) { - let currentOperation = pipelineListItems[i].querySelector('.operationName').textContent; - let nextOperation = pipelineListItems[i + 1].querySelector('.operationName').textContent; - if (currentOperation === '/add-password') { - containsAddPassword = true; - } - - let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; - let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; - - // Strip off 'ZIP-' prefix - currentOperationDescription = currentOperationDescription.replace("ZIP-", ''); - nextOperationDescription = nextOperationDescription.replace("ZIP-", ''); - - let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; - let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; - - // Splitting in case of multiple possible output/input - let currentOperationOutputArr = currentOperationOutput.split('/'); - let nextOperationInputArr = nextOperationInput.split('/'); - - if (currentOperationOutput !== 'ANY' && nextOperationInput !== 'ANY') { - let intersection = currentOperationOutputArr.filter(value => nextOperationInputArr.includes(value)); - console.log(`Intersection: ${intersection}`); - - if (intersection.length === 0) { - updateValidateButton(false); - isValid = false; - console.log(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); - alert(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); - break; - } - } - } - if (containsAddPassword && pipelineListItems[pipelineListItems.length - 1].querySelector('.operationName').textContent !== '/add-password') { - updateValidateButton(false); - alert('The "add-password" operation should be at the end of the operations sequence. Please adjust the operations order.'); - return false; - } - if (isValid) { - console.log('Pipeline is valid'); - // Continue with the pipeline operation - } else { - console.error('Pipeline is not valid'); - // Stop operation, maybe display an error to the user - } - updateValidateButton(isValid); - return isValid; -} - -function updateValidateButton(isValid) { - var validateButton = document.getElementById('validateButton'); - if (isValid) { - validateButton.classList.remove('btn-danger'); - validateButton.classList.add('btn-success'); - } else { - validateButton.classList.remove('btn-success'); - validateButton.classList.add('btn-danger'); - } -} - - - - -document.getElementById('submitConfigBtn').addEventListener('click', function() { - - if (validatePipeline() === false) { - return; - } - let selectedOperation = document.getElementById('operationsDropdown').value; - - - - var pipelineName = document.getElementById('pipelineName').value; - let pipelineList = document.getElementById('pipelineList').children; - let pipelineConfig = { - "name": pipelineName, - "pipeline": [], - "_examples": { - "outputDir": "{outputFolder}/{folderName}", - "outputFileName": "{filename}-{pipelineName}-{date}-{time}" - }, - "outputDir": "httpWebRequest", - "outputFileName": "{filename}" - }; - - for (let i = 0; i < pipelineList.length; i++) { - let operationName = pipelineList[i].querySelector('.operationName').textContent; - let parameters = operationSettings[operationName] || {}; - - pipelineConfig.pipeline.push({ - "operation": operationName, - "parameters": parameters - }); - } - - - - - - - - - - - - - - let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2); - - let formData = new FormData(); - - let fileInput = document.getElementById('fileInput-input'); - let files = fileInput.files; - - for (let i = 0; i < files.length; i++) { - console.log("files[i]", files[i].name); - formData.append('fileInput', files[i], files[i].name); - } - - console.log("pipelineConfigJson", pipelineConfigJson); - formData.append('json', pipelineConfigJson); - console.log("formData", formData); - - fetch('api/v1/pipeline/handleData', { - method: 'POST', - body: formData - }) - .then(response => { - // Save the response to use it later - const responseToUseLater = response; - - return response.blob().then(blob => { - let url = window.URL.createObjectURL(blob); - let a = document.createElement('a'); - a.href = url; - - // Use responseToUseLater instead of response - const contentDisposition = responseToUseLater.headers.get('Content-Disposition'); - let filename = 'download'; - if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) { - filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim(); - } - a.download = filename; - - document.body.appendChild(a); - a.click(); - a.remove(); - }); - }) - .catch((error) => { - console.error('Error:', error); + document.getElementById('validateButton').addEventListener('click', function(event) { + event.preventDefault(); + validatePipeline(); }); - -}); - -let apiDocs = {}; -let apiSchemas = {}; -let operationSettings = {}; - -fetch('v1/api-docs') - .then(response => response.json()) - .then(data => { - - apiDocs = data.paths; - apiSchemas = data.components.schemas; - let operationsDropdown = document.getElementById('operationsDropdown'); - const ignoreOperations = ["/api/v1/pipeline/handleData", "/api/v1/pipeline/operationToIgnore"]; // Add the operations you want to ignore here - - operationsDropdown.innerHTML = ''; - - let operationsByTag = {}; - - // Group operations by tags - Object.keys(data.paths).forEach(operationPath => { - let operation = data.paths[operationPath].post; - if(!operation || !operation.description) { - console.log(operationPath); + function validatePipeline() { + let pipelineListItems = document.getElementById('pipelineList').children; + let isValid = true; + let containsAddPassword = false; + for (let i = 0; i < pipelineListItems.length - 1; i++) { + let currentOperation = pipelineListItems[i].querySelector('.operationName').textContent; + let nextOperation = pipelineListItems[i + 1].querySelector('.operationName').textContent; + if (currentOperation === '/add-password') { + containsAddPassword = true; } - if (operation && !ignoreOperations.includes(operationPath) && !operation.description.includes("Type:MISO")) { - let operationTag = operation.tags[0]; // This assumes each operation has exactly one tag - if (!operationsByTag[operationTag]) { - operationsByTag[operationTag] = []; + + let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; + let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; + + // Strip off 'ZIP-' prefix + currentOperationDescription = currentOperationDescription.replace("ZIP-", ''); + nextOperationDescription = nextOperationDescription.replace("ZIP-", ''); + + let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; + let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; + + // Splitting in case of multiple possible output/input + let currentOperationOutputArr = currentOperationOutput.split('/'); + let nextOperationInputArr = nextOperationInput.split('/'); + + if (currentOperationOutput !== 'ANY' && nextOperationInput !== 'ANY') { + let intersection = currentOperationOutputArr.filter(value => nextOperationInputArr.includes(value)); + console.log(`Intersection: ${intersection}`); + + if (intersection.length === 0) { + updateValidateButton(false); + isValid = false; + console.log(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); + alert(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); + break; } - operationsByTag[operationTag].push(operationPath); } - }); - // Specify the order of tags - let tagOrder = ["General", "Security", "Convert", "Misc", "Filter"]; - - // Create dropdown options - tagOrder.forEach(tag => { - if (operationsByTag[tag]) { - let group = document.createElement('optgroup'); - group.label = tag; - - operationsByTag[tag].forEach(operationPath => { - let option = document.createElement('option'); - - let operationPathDisplay = operationPath - operationPathDisplay = operationPath.replace(new RegExp("api/v1/" + tag.toLowerCase() + "/", 'i'), ""); - - - if(operationPath.includes("/convert")){ - operationPathDisplay = operationPathDisplay.replace(/^\//, '').replaceAll("/", " to "); - } else { - operationPathDisplay = operationPathDisplay.replace(/\//g, ''); // Remove slashes - } - operationPathDisplay = operationPathDisplay.replaceAll(" ","-"); - option.textContent = operationPathDisplay; - option.value = operationPath; // Keep the value with slashes for querying - group.appendChild(option); - }); - - operationsDropdown.appendChild(group); - } - }); - }); - - -document.getElementById('addOperationBtn').addEventListener('click', function() { - let selectedOperation = document.getElementById('operationsDropdown').value; - let pipelineList = document.getElementById('pipelineList'); - - let listItem = document.createElement('li'); - listItem.className = "list-group-item"; - let hasSettings = false; - if (apiDocs[selectedOperation] && apiDocs[selectedOperation].post) { - const postMethod = apiDocs[selectedOperation].post; - - // Check if parameters exist - if (postMethod.parameters && postMethod.parameters.length > 0) { - hasSettings = true; - } else if (postMethod.requestBody && postMethod.requestBody.content['multipart/form-data']) { - // Extract the reference key - const refKey = postMethod.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); - // Check if the referenced schema exists and has properties - if (apiSchemas[refKey] && Object.keys(apiSchemas[refKey].properties).length > 0) { - hasSettings = true; - } - } + } + if (containsAddPassword && pipelineListItems[pipelineListItems.length - 1].querySelector('.operationName').textContent !== '/add-password') { + updateValidateButton(false); + alert('The "add-password" operation should be at the end of the operations sequence. Please adjust the operations order.'); + return false; + } + if (isValid) { + console.log('Pipeline is valid'); + // Continue with the pipeline operation + } else { + console.error('Pipeline is not valid'); + // Stop operation, maybe display an error to the user + } + updateValidateButton(isValid); + return isValid; } - - - - - listItem.innerHTML = ` -
-
${selectedOperation}
-
- - - - -
-
-`; - - - pipelineList.appendChild(listItem); - - listItem.querySelector('.move-up').addEventListener('click', function(event) { - event.preventDefault(); - if (listItem.previousElementSibling) { - pipelineList.insertBefore(listItem, listItem.previousElementSibling); + + function updateValidateButton(isValid) { + var validateButton = document.getElementById('validateButton'); + if (isValid) { + validateButton.classList.remove('btn-danger'); + validateButton.classList.add('btn-success'); + } else { + validateButton.classList.remove('btn-success'); + validateButton.classList.add('btn-danger'); } - }); - - listItem.querySelector('.move-down').addEventListener('click', function(event) { - event.preventDefault(); - if (listItem.nextElementSibling) { - pipelineList.insertBefore(listItem.nextElementSibling, listItem); + } + + + + + document.getElementById('submitConfigBtn').addEventListener('click', function() { + + if (validatePipeline() === false) { + return; } - }); - - listItem.querySelector('.remove').addEventListener('click', function(event) { - event.preventDefault(); - pipelineList.removeChild(listItem); - hideOrShowPipelineHeader(); - }); - - listItem.querySelector('.pipelineSettings').addEventListener('click', function(event) { - event.preventDefault(); - showpipelineSettingsModal(selectedOperation); - hideOrShowPipelineHeader(); - }); - - function showpipelineSettingsModal(operation) { - let pipelineSettingsModal = document.getElementById('pipelineSettingsModal'); - let pipelineSettingsContent = document.getElementById('pipelineSettingsContent'); - let operationData = apiDocs[operation].post.parameters || []; - - // Resolve the $ref reference to get actual schema properties - let refKey = apiDocs[operation].post.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); - let requestBodyData = apiSchemas[refKey].properties || {}; - - // Combine operationData and requestBodyData into a single array - operationData = operationData.concat(Object.keys(requestBodyData).map(key => ({ - name: key, - schema: requestBodyData[key] - }))); - - pipelineSettingsContent.innerHTML = ''; - - operationData.forEach(parameter => { - // If the parameter name is 'fileInput', return early to skip the rest of this iteration - if (parameter.name === 'fileInput') return; - - let parameterDiv = document.createElement('div'); - parameterDiv.className = "mb-3"; - - let parameterLabel = document.createElement('label'); - parameterLabel.textContent = `${parameter.name} (${parameter.schema.type}): `; - parameterLabel.title = parameter.schema.description; - parameterLabel.setAttribute('for', parameter.name); - parameterDiv.appendChild(parameterLabel); - - let defaultValue = parameter.schema.example; - if (defaultValue === undefined) defaultValue = parameter.schema.default; - - let parameterInput; - - // check if enum exists in schema - if (parameter.schema.enum) { - // if enum exists, create a select element - parameterInput = document.createElement('select'); - parameterInput.className = "form-control"; - - // iterate over each enum value and create an option for it - parameter.schema.enum.forEach(value => { - let option = document.createElement('option'); - option.value = value; - option.text = value; - parameterInput.appendChild(option); + let selectedOperation = document.getElementById('operationsDropdown').value; + + + + var pipelineName = document.getElementById('pipelineName').value; + let pipelineList = document.getElementById('pipelineList').children; + let pipelineConfig = { + "name": pipelineName, + "pipeline": [], + "_examples": { + "outputDir": "{outputFolder}/{folderName}", + "outputFileName": "{filename}-{pipelineName}-{date}-{time}" + }, + "outputDir": "httpWebRequest", + "outputFileName": "{filename}" + }; + + for (let i = 0; i < pipelineList.length; i++) { + let operationName = pipelineList[i].querySelector('.operationName').textContent; + let parameters = operationSettings[operationName] || {}; + + pipelineConfig.pipeline.push({ + "operation": operationName, + "parameters": parameters + }); + } + + + + + + + + + + + + + + let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2); + + let formData = new FormData(); + + let fileInput = document.getElementById('fileInput-input'); + let files = fileInput.files; + + for (let i = 0; i < files.length; i++) { + console.log("files[i]", files[i].name); + formData.append('fileInput', files[i], files[i].name); + } + + console.log("pipelineConfigJson", pipelineConfigJson); + formData.append('json', pipelineConfigJson); + console.log("formData", formData); + + fetch('api/v1/pipeline/handleData', { + method: 'POST', + body: formData + }) + .then(response => { + // Save the response to use it later + const responseToUseLater = response; + + return response.blob().then(blob => { + let url = window.URL.createObjectURL(blob); + let a = document.createElement('a'); + a.href = url; + + // Use responseToUseLater instead of response + const contentDisposition = responseToUseLater.headers.get('Content-Disposition'); + let filename = 'download'; + if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) { + filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim(); + } + a.download = filename; + + document.body.appendChild(a); + a.click(); + a.remove(); }); - } else { - // switch-case statement for handling non-enum types - switch (parameter.schema.type) { - case 'string': - if (parameter.schema.format === 'binary') { - // This is a file input - - //parameterInput = document.createElement('input'); - //parameterInput.type = 'file'; - //parameterInput.className = "form-control"; - - parameterInput = document.createElement('input'); - parameterInput.type = 'text'; - parameterInput.className = "form-control"; - parameterInput.value = "FileInputPathToBeInputtedManuallyOffline"; + }) + .catch((error) => { + console.error('Error:', error); + }); + + }); + + let apiDocs = {}; + let apiSchemas = {}; + let operationSettings = {}; + + fetch('v1/api-docs') + .then(response => response.json()) + .then(data => { + + apiDocs = data.paths; + apiSchemas = data.components.schemas; + let operationsDropdown = document.getElementById('operationsDropdown'); + const ignoreOperations = ["/api/v1/pipeline/handleData", "/api/v1/pipeline/operationToIgnore"]; // Add the operations you want to ignore here + + operationsDropdown.innerHTML = ''; + + let operationsByTag = {}; + + // Group operations by tags + Object.keys(data.paths).forEach(operationPath => { + let operation = data.paths[operationPath].post; + if (!operation || !operation.description) { + console.log(operationPath); + } + //!operation.description.includes("Type:MISO") + if (operation && !ignoreOperations.includes(operationPath)) { + let operationTag = operation.tags[0]; // This assumes each operation has exactly one tag + if (!operationsByTag[operationTag]) { + operationsByTag[operationTag] = []; + } + operationsByTag[operationTag].push(operationPath); + } + }); + // Specify the order of tags + let tagOrder = ["General", "Security", "Convert", "Misc", "Filter"]; + + // Create dropdown options + tagOrder.forEach(tag => { + if (operationsByTag[tag]) { + let group = document.createElement('optgroup'); + group.label = tag; + + operationsByTag[tag].forEach(operationPath => { + let option = document.createElement('option'); + + let operationPathDisplay = operationPath + operationPathDisplay = operationPath.replace(new RegExp("api/v1/" + tag.toLowerCase() + "/", 'i'), ""); + + + if (operationPath.includes("/convert")) { + operationPathDisplay = operationPathDisplay.replace(/^\//, '').replaceAll("/", " to "); } else { + operationPathDisplay = operationPathDisplay.replace(/\//g, ''); // Remove slashes + } + operationPathDisplay = operationPathDisplay.replaceAll(" ", "-"); + option.textContent = operationPathDisplay; + option.value = operationPath; // Keep the value with slashes for querying + group.appendChild(option); + }); + + operationsDropdown.appendChild(group); + } + }); + }); + + + document.getElementById('addOperationBtn').addEventListener('click', function() { + let selectedOperation = document.getElementById('operationsDropdown').value; + let pipelineList = document.getElementById('pipelineList'); + + let listItem = document.createElement('li'); + listItem.className = "list-group-item"; + let hasSettings = false; + if (apiDocs[selectedOperation] && apiDocs[selectedOperation].post) { + const postMethod = apiDocs[selectedOperation].post; + + // Check if parameters exist + if (postMethod.parameters && postMethod.parameters.length > 0) { + hasSettings = true; + } else if (postMethod.requestBody && postMethod.requestBody.content['multipart/form-data']) { + // Extract the reference key + const refKey = postMethod.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); + // Check if the referenced schema exists and has properties + if (apiSchemas[refKey] && Object.keys(apiSchemas[refKey].properties).length > 0) { + hasSettings = true; + } + } + } + + + + + listItem.innerHTML = ` +
+
${selectedOperation}
+
+ + + + +
+
+ `; + + + pipelineList.appendChild(listItem); + + listItem.querySelector('.move-up').addEventListener('click', function(event) { + event.preventDefault(); + if (listItem.previousElementSibling) { + pipelineList.insertBefore(listItem, listItem.previousElementSibling); + updateConfigInDropdown(); + } + }); + + listItem.querySelector('.move-down').addEventListener('click', function(event) { + event.preventDefault(); + if (listItem.nextElementSibling) { + pipelineList.insertBefore(listItem.nextElementSibling, listItem); + updateConfigInDropdown(); + } + + }); + + listItem.querySelector('.remove').addEventListener('click', function(event) { + event.preventDefault(); + pipelineList.removeChild(listItem); + hideOrShowPipelineHeader(); + updateConfigInDropdown(); + }); + + listItem.querySelector('.pipelineSettings').addEventListener('click', function(event) { + event.preventDefault(); + showpipelineSettingsModal(selectedOperation); + hideOrShowPipelineHeader(); + }); + + function showpipelineSettingsModal(operation) { + let pipelineSettingsModal = document.getElementById('pipelineSettingsModal'); + let pipelineSettingsContent = document.getElementById('pipelineSettingsContent'); + let operationData = apiDocs[operation].post.parameters || []; + + // Resolve the $ref reference to get actual schema properties + let refKey = apiDocs[operation].post.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); + let requestBodyData = apiSchemas[refKey].properties || {}; + + // Combine operationData and requestBodyData into a single array + operationData = operationData.concat(Object.keys(requestBodyData).map(key => ({ + name: key, + schema: requestBodyData[key] + }))); + + pipelineSettingsContent.innerHTML = ''; + + operationData.forEach(parameter => { + // If the parameter name is 'fileInput', return early to skip the rest of this iteration + if (parameter.name === 'fileInput') return; + + let parameterDiv = document.createElement('div'); + parameterDiv.className = "mb-3"; + + let parameterLabel = document.createElement('label'); + parameterLabel.textContent = `${parameter.name} (${parameter.schema.type}): `; + parameterLabel.title = parameter.schema.description; + parameterLabel.setAttribute('for', parameter.name); + parameterDiv.appendChild(parameterLabel); + + let defaultValue = parameter.schema.example; + if (defaultValue === undefined) defaultValue = parameter.schema.default; + + let parameterInput; + + // check if enum exists in schema + if (parameter.schema.enum) { + // if enum exists, create a select element + parameterInput = document.createElement('select'); + parameterInput.className = "form-control"; + + // iterate over each enum value and create an option for it + parameter.schema.enum.forEach(value => { + let option = document.createElement('option'); + option.value = value; + option.text = value; + parameterInput.appendChild(option); + }); + } else { + // switch-case statement for handling non-enum types + switch (parameter.schema.type) { + case 'string': + if (parameter.schema.format === 'binary') { + // This is a file input + + //parameterInput = document.createElement('input'); + //parameterInput.type = 'file'; + //parameterInput.className = "form-control"; + + parameterInput = document.createElement('input'); + parameterInput.type = 'text'; + parameterInput.className = "form-control"; + parameterInput.value = "FileInputPathToBeInputtedManuallyOffline"; + } else { + parameterInput = document.createElement('input'); + parameterInput.type = 'text'; + parameterInput.className = "form-control"; + if (defaultValue !== undefined) parameterInput.value = defaultValue; + } + break; + case 'number': + case 'integer': + parameterInput = document.createElement('input'); + parameterInput.type = 'number'; + parameterInput.className = "form-control"; + if (defaultValue !== undefined) parameterInput.value = defaultValue; + break; + case 'boolean': + parameterInput = document.createElement('input'); + parameterInput.type = 'checkbox'; + if (defaultValue === true) parameterInput.checked = true; + break; + case 'array': + case 'object': + parameterInput = document.createElement('textarea'); + parameterInput.placeholder = `Enter a JSON formatted ${parameter.schema.type}`; + parameterInput.className = "form-control"; + break; + default: parameterInput = document.createElement('input'); parameterInput.type = 'text'; parameterInput.className = "form-control"; if (defaultValue !== undefined) parameterInput.value = defaultValue; - } - break; - case 'number': - case 'integer': - parameterInput = document.createElement('input'); - parameterInput.type = 'number'; - parameterInput.className = "form-control"; - if (defaultValue !== undefined) parameterInput.value = defaultValue; - break; - case 'boolean': - parameterInput = document.createElement('input'); - parameterInput.type = 'checkbox'; - if (defaultValue === true) parameterInput.checked = true; - break; - case 'array': - case 'object': - parameterInput = document.createElement('textarea'); - parameterInput.placeholder = `Enter a JSON formatted ${parameter.schema.type}`; - parameterInput.className = "form-control"; - break; - default: - parameterInput = document.createElement('input'); - parameterInput.type = 'text'; - parameterInput.className = "form-control"; - if (defaultValue !== undefined) parameterInput.value = defaultValue; + } } - } - parameterInput.id = parameter.name; - - console.log("defaultValue", defaultValue); - console.log("parameterInput", parameterInput); - if (operationSettings[operation] && operationSettings[operation][parameter.name] !== undefined) { - let savedValue = operationSettings[operation][parameter.name]; - - switch (parameter.schema.type) { - case 'number': - case 'integer': - parameterInput.value = savedValue.toString(); - break; - case 'boolean': - parameterInput.checked = savedValue; - break; - case 'array': - case 'object': - parameterInput.value = JSON.stringify(savedValue); - break; - default: - parameterInput.value = savedValue; - } - } - console.log("parameterInput2", parameterInput); - parameterDiv.appendChild(parameterInput); - - pipelineSettingsContent.appendChild(parameterDiv); - }); - - let saveButton = document.createElement('button'); - saveButton.textContent = "Save Settings"; - saveButton.className = "btn btn-primary"; - saveButton.addEventListener('click', function(event) { - event.preventDefault(); - let settings = {}; - operationData.forEach(parameter => { - if(parameter.name !== "fileInput"){ - let value = document.getElementById(parameter.name).value; + parameterInput.id = parameter.name; + + console.log("defaultValue", defaultValue); + console.log("parameterInput", parameterInput); + if (operationSettings[operation] && operationSettings[operation][parameter.name] !== undefined) { + let savedValue = operationSettings[operation][parameter.name]; + switch (parameter.schema.type) { case 'number': case 'integer': - settings[parameter.name] = Number(value); + parameterInput.value = savedValue.toString(); break; case 'boolean': - settings[parameter.name] = document.getElementById(parameter.name).checked; + parameterInput.checked = savedValue; break; case 'array': case 'object': - try { - settings[parameter.name] = JSON.parse(value); - } catch (err) { - console.error(`Invalid JSON format for ${parameter.name}`); - } + parameterInput.value = JSON.stringify(savedValue); break; default: - settings[parameter.name] = value; + parameterInput.value = savedValue; } } + console.log("parameterInput2", parameterInput); + parameterDiv.appendChild(parameterInput); + + pipelineSettingsContent.appendChild(parameterDiv); }); - operationSettings[operation] = settings; - //pipelineSettingsModal.style.display = "none"; - }); - pipelineSettingsContent.appendChild(saveButton); - - //pipelineSettingsModal.style.display = "block"; - - //pipelineSettingsModal.getElementsByClassName("close")[0].onclick = function() { - // pipelineSettingsModal.style.display = "none"; - //} - - //window.onclick = function(event) { - // if (event.target == pipelineSettingsModal) { - // pipelineSettingsModal.style.display = "none"; - // } - //} + + let saveButton = document.createElement('button'); + saveButton.textContent = "Save Settings"; + saveButton.className = "btn btn-primary"; + saveButton.addEventListener('click', function(event) { + event.preventDefault(); + let settings = {}; + operationData.forEach(parameter => { + if (parameter.name !== "fileInput") { + let value = document.getElementById(parameter.name).value; + switch (parameter.schema.type) { + case 'number': + case 'integer': + settings[parameter.name] = Number(value); + break; + case 'boolean': + settings[parameter.name] = document.getElementById(parameter.name).checked; + break; + case 'array': + case 'object': + try { + settings[parameter.name] = JSON.parse(value); + } catch (err) { + console.error(`Invalid JSON format for ${parameter.name}`); + } + break; + default: + settings[parameter.name] = value; + } + } + }); + operationSettings[operation] = settings; + //pipelineSettingsModal.style.display = "none"; + }); + pipelineSettingsContent.appendChild(saveButton); + + //pipelineSettingsModal.style.display = "block"; + + //pipelineSettingsModal.getElementsByClassName("close")[0].onclick = function() { + // pipelineSettingsModal.style.display = "none"; + //} + + //window.onclick = function(event) { + // if (event.target == pipelineSettingsModal) { + // pipelineSettingsModal.style.display = "none"; + // } + //} + } + updateConfigInDropdown(); + hideOrShowPipelineHeader(); + }); + + function updateConfigInDropdown() { + let pipelineSelect = document.getElementById('pipelineSelect'); + let selectedOption = pipelineSelect.options[pipelineSelect.selectedIndex]; + + // Get the current configuration as JSON + let pipelineConfigJson = configToJson(); + console.log("pipelineConfigJson", pipelineConfigJson); + if (!pipelineConfigJson) { + console.error("Failed to update configuration: Invalid configuration"); + return; + } + + // Update the value of the selected option with the new configuration + selectedOption.value = pipelineConfigJson; + } - hideOrShowPipelineHeader(); -}); - - var saveBtn = document.getElementById('savePipelineBtn'); - + // Remove any existing event listeners saveBtn.removeEventListener('click', savePipeline); // Add the event listener saveBtn.addEventListener('click', savePipeline); console.log("saveBtn", saveBtn) - function savePipeline() { - - if (validatePipeline() === false) { - return; + + function configToJson() { + if (!validatePipeline()) { + return null; // Return null if validation fails } - + var pipelineName = document.getElementById('pipelineName').value; let pipelineList = document.getElementById('pipelineList').children; let pipelineConfig = { @@ -497,35 +518,49 @@ document.getElementById('addOperationBtn').addEventListener('click', function() "outputDir": "{outputFolder}", "outputFileName": "{filename}" }; - + for (let i = 0; i < pipelineList.length; i++) { let operationName = pipelineList[i].querySelector('.operationName').textContent; let parameters = operationSettings[operationName] || {}; - + parameters['fileInput'] = 'automated'; - + pipelineConfig.pipeline.push({ "operation": operationName, "parameters": parameters }); } - console.log("Downloading.."); + + return JSON.stringify(pipelineConfig, null, 2); + } + + + + function savePipeline() { + let pipelineConfigJson = configToJson(); + if (!pipelineConfigJson) { + console.error("Failed to save pipeline: Invalid configuration"); + return; + } + + let pipelineName = document.getElementById('pipelineName').value; + console.log("Downloading..."); let a = document.createElement('a'); - a.href = URL.createObjectURL(new Blob([JSON.stringify(pipelineConfig, null, 2)], { - type: 'application/json' - })); + a.href = URL.createObjectURL(new Blob([pipelineConfigJson], { type: 'application/json' })); a.download = pipelineName + '.json'; a.style.display = 'none'; - + document.body.appendChild(a); a.click(); document.body.removeChild(a); } - + + async function processPipelineConfig(configString) { + console.log("configString",configString); let pipelineConfig = JSON.parse(configString); let pipelineList = document.getElementById('pipelineList'); - + while (pipelineList.firstChild) { pipelineList.removeChild(pipelineList.firstChild); } @@ -534,15 +569,15 @@ document.getElementById('addOperationBtn').addEventListener('click', function() let operationsDropdown = document.getElementById('operationsDropdown'); operationsDropdown.value = operationConfig.operation; operationSettings[operationConfig.operation] = operationConfig.parameters; - + // assuming addOperation is async await new Promise((resolve) => { document.getElementById('addOperationBtn').addEventListener('click', resolve, { once: true }); document.getElementById('addOperationBtn').click(); }); - + let lastOperation = pipelineList.lastChild; - + Object.keys(operationConfig.parameters).forEach(parameterName => { let input = document.getElementById(parameterName); if (input) { @@ -559,7 +594,7 @@ document.getElementById('addOperationBtn').addEventListener('click', function() let newInput = document.createElement('input'); newInput.type = 'file'; newInput.id = parameterName; - + // Add the new file input to the main page (change the selector according to your needs) document.querySelector('#main').appendChild(newInput); } @@ -571,15 +606,15 @@ document.getElementById('addOperationBtn').addEventListener('click', function() } } }); - + } } - - + + document.getElementById('uploadPipelineBtn').addEventListener('click', function() { document.getElementById('uploadPipelineInput').click(); }); - + document.getElementById('uploadPipelineInput').addEventListener('change', function(e) { let reader = new FileReader(); reader.onload = function(event) { @@ -588,22 +623,22 @@ document.getElementById('addOperationBtn').addEventListener('click', function() reader.readAsText(e.target.files[0]); hideOrShowPipelineHeader(); }); - + document.getElementById('pipelineSelect').addEventListener('change', function(e) { let selectedPipelineJson = e.target.value; // assuming the selected value is the JSON string of the pipeline config processPipelineConfig(selectedPipelineJson); }); - - - function hideOrShowPipelineHeader() { - var pipelineHeader = document.getElementById('pipelineHeader'); - var pipelineList = document.getElementById('pipelineList'); - if (pipelineList.children.length === 0) { - // Hide the pipeline header if there are no items in the pipeline list - pipelineHeader.style.display = 'none'; - } else { - // Show the pipeline header if there are items in the pipeline list - pipelineHeader.style.display = 'block'; - } + + function hideOrShowPipelineHeader() { + var pipelineHeader = document.getElementById('pipelineHeader'); + var pipelineList = document.getElementById('pipelineList'); + + if (pipelineList.children.length === 0) { + // Hide the pipeline header if there are no items in the pipeline list + pipelineHeader.style.display = 'none'; + } else { + // Show the pipeline header if there are items in the pipeline list + pipelineHeader.style.display = 'block'; + } } diff --git a/src/main/resources/templates/pipeline.html b/src/main/resources/templates/pipeline.html index 51caf186..898976c7 100644 --- a/src/main/resources/templates/pipeline.html +++ b/src/main/resources/templates/pipeline.html @@ -48,6 +48,8 @@
- - - - - - -
-
-
-
-
- -
-
- - - -

Current Limitations

- - -

How it Works Notes

- - -

How to use pre-load configs in web UI

- - -

Todo

- - - -

User Guide for Local Directory Scanning and File Processing

- -

Setting Up Watched Folders:

-

Create a folder where you want your files to be monitored. This is your 'watched folder'.

-

The default directory for this is ./pipeline/watchedFolders/

-

Place any directories you want to be scanned into this folder, this folder should contain multiple folders each for their own tasks and pipelines.

- -

Configuring Processing with JSON Files:

-

In each directory you want processed (e.g ./pipeline/watchedFolders/officePrinter), include a JSON configuration file.

-

This JSON file should specify how you want the files in the directory to be handled (e.g., what operations to perform on them) which can be made, configured and downloaded from Stirling-PDF Pipeline interface.

- -

Automatic Scanning and Processing:

-

The system automatically checks the watched folder every minute for new directories and files to process.

-

When a directory with a valid JSON configuration file is found, it begins processing the files inside as per the configuration.

- -

Processing Steps:

-

Files in each directory are processed according to the instructions in the JSON file.

-

This might involve file conversions, data filtering, renaming files, etc. If the output of a step is a zip, this zip will be automatically unzipped as it passes to next process.

- -

Results and Output:

-

After processing, the results are saved in a specified output location. This could be a different folder or location as defined in the JSON file or the default location ./pipeline/finishedFolders/.

-

Each processed file is named and organized according to the rules set in the JSON configuration.

- -

Completion and Cleanup:

-

Once processing is complete, the original files in the watched folder's directory are removed.

-

You can find the processed files in the designated output location.

- -

Error Handling:

-

If there's an error during processing, the system will not delete the original files, allowing you to check and retry if necessary.

- -

User Interaction:

-

As a user, your main tasks are to set up the watched folders, place directories with files for processing, and create the corresponding JSON configuration files.

-

The system handles the rest, including scanning, processing, and outputting results.

- - - - From 39045df7858d31076530f70f7c58e27101156ede Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 31 Dec 2023 13:34:35 +0000 Subject: [PATCH 08/23] formats --- .../security/SecurityConfiguration.java | 5 +- .../security/UserAuthenticationFilter.java | 20 +- .../api/pipeline/PipelineProcessor.java | 238 +++++++++--------- .../controller/web/GeneralWebController.java | 113 +++++---- .../software/SPDF/utils/RequestUriUtils.java | 19 +- 5 files changed, 201 insertions(+), 194 deletions(-) diff --git a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java index a2908356..bf1e3661 100644 --- a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java +++ b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java @@ -102,8 +102,9 @@ public class SecurityConfiguration { || trimmedUri.startsWith("/images/") || trimmedUri.startsWith("/public/") || trimmedUri.startsWith("/css/") - || trimmedUri.startsWith("/js/") || - trimmedUri.startsWith("/api/v1/info/status"); + || trimmedUri.startsWith("/js/") + || trimmedUri.startsWith( + "/api/v1/info/status"); }) .permitAll() .anyRequest() diff --git a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java index 2a4c68d5..d12fc72b 100644 --- a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java +++ b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java @@ -95,16 +95,16 @@ public class UserAuthenticationFilter extends OncePerRequestFilter { String uri = request.getRequestURI(); String contextPath = request.getContextPath(); String[] permitAllPatterns = { - contextPath + "/login", - contextPath + "/register", - contextPath + "/error", - contextPath + "/images/", - contextPath + "/public/", - contextPath + "/css/", - contextPath + "/js/", - contextPath + "/pdfjs/", - contextPath + "/api/v1/info/status", - contextPath + "/site.webmanifest" + contextPath + "/login", + contextPath + "/register", + contextPath + "/error", + contextPath + "/images/", + contextPath + "/public/", + contextPath + "/css/", + contextPath + "/js/", + contextPath + "/pdfjs/", + contextPath + "/api/v1/info/status", + contextPath + "/site.webmanifest" }; for (String pattern : permitAllPatterns) { diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java index f87cf118..9bebb96f 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java @@ -49,145 +49,153 @@ public class PipelineProcessor { @Autowired(required = false) private UserServiceInterface userService; - @Autowired - private ServletContext servletContext; + @Autowired private ServletContext servletContext; - + private String getApiKeyForUser() { + if (userService == null) return ""; + return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId()); + } + private String getBaseUrl() { + String contextPath = servletContext.getContextPath(); + String port = SPdfApplication.getPort(); - private String getApiKeyForUser() { - if (userService == null) - return ""; - return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId()); - } + return "http://localhost:" + port + contextPath + "/"; + } + List runPipelineAgainstFiles(List outputFiles, PipelineConfig config) + throws Exception { - private String getBaseUrl() { - String contextPath = servletContext.getContextPath(); - String port = SPdfApplication.getPort(); + ByteArrayOutputStream logStream = new ByteArrayOutputStream(); + PrintStream logPrintStream = new PrintStream(logStream); - return "http://localhost:" + port + contextPath + "/"; - } + boolean hasErrors = false; - - - List runPipelineAgainstFiles(List outputFiles, PipelineConfig config) throws Exception { + for (PipelineOperation pipelineOperation : config.getOperations()) { + String operation = pipelineOperation.getOperation(); + boolean isMultiInputOperation = apiDocService.isMultiInput(operation); - ByteArrayOutputStream logStream = new ByteArrayOutputStream(); - PrintStream logPrintStream = new PrintStream(logStream); + logger.info( + "Running operation: {} isMultiInputOperation {}", + operation, + isMultiInputOperation); + Map parameters = pipelineOperation.getParameters(); + String inputFileExtension = ""; - boolean hasErrors = false; + // TODO + // if (operationNode.has("inputFileType")) { + // inputFileExtension = operationNode.get("inputFileType").asText(); + // } else { + inputFileExtension = ".pdf"; + // } + final String finalInputFileExtension = inputFileExtension; - for (PipelineOperation pipelineOperation : config.getOperations()) { - String operation = pipelineOperation.getOperation(); - boolean isMultiInputOperation = apiDocService.isMultiInput(operation); + String url = getBaseUrl() + operation; - logger.info("Running operation: {} isMultiInputOperation {}", operation, isMultiInputOperation); - Map parameters = pipelineOperation.getParameters(); - String inputFileExtension = ""; - - //TODO - //if (operationNode.has("inputFileType")) { - // inputFileExtension = operationNode.get("inputFileType").asText(); - //} else { - inputFileExtension = ".pdf"; - //} - final String finalInputFileExtension = inputFileExtension; - - String url = getBaseUrl() + operation; - - List newOutputFiles = new ArrayList<>(); - if (!isMultiInputOperation) { - for (Resource file : outputFiles) { - boolean hasInputFileType = false; - if (file.getFilename().endsWith(inputFileExtension)) { - hasInputFileType = true; - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("fileInput", file); + List newOutputFiles = new ArrayList<>(); + if (!isMultiInputOperation) { + for (Resource file : outputFiles) { + boolean hasInputFileType = false; + if (file.getFilename().endsWith(inputFileExtension)) { + hasInputFileType = true; + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("fileInput", file); - - for(Entry entry : parameters.entrySet()) { - body.add(entry.getKey(), entry.getValue()); - } + for (Entry entry : parameters.entrySet()) { + body.add(entry.getKey(), entry.getValue()); + } - ResponseEntity response = sendWebRequest(url, body); + ResponseEntity response = sendWebRequest(url, body); - // If the operation is filter and the response body is null or empty, skip this - // file - if (operation.startsWith("filter-") - && (response.getBody() == null || response.getBody().length == 0)) { - logger.info("Skipping file due to failing {}", operation); - continue; - } + // If the operation is filter and the response body is null or empty, skip + // this + // file + if (operation.startsWith("filter-") + && (response.getBody() == null || response.getBody().length == 0)) { + logger.info("Skipping file due to failing {}", operation); + continue; + } - if (!response.getStatusCode().equals(HttpStatus.OK)) { - logPrintStream.println("Error: " + response.getBody()); - hasErrors = true; - continue; - } - processOutputFiles(operation, file.getFilename(), response, newOutputFiles); - - } + if (!response.getStatusCode().equals(HttpStatus.OK)) { + logPrintStream.println("Error: " + response.getBody()); + hasErrors = true; + continue; + } + processOutputFiles(operation, file.getFilename(), response, newOutputFiles); + } - if (!hasInputFileType) { - logPrintStream.println( - "No files with extension " + inputFileExtension + " found for operation " + operation); - hasErrors = true; - } + if (!hasInputFileType) { + logPrintStream.println( + "No files with extension " + + inputFileExtension + + " found for operation " + + operation); + hasErrors = true; + } + } - - } + } else { + // Filter and collect all files that match the inputFileExtension + List matchingFiles = + outputFiles.stream() + .filter( + file -> + file.getFilename() + .endsWith(finalInputFileExtension)) + .collect(Collectors.toList()); - } else { - // Filter and collect all files that match the inputFileExtension - List matchingFiles = outputFiles.stream() - .filter(file -> file.getFilename().endsWith(finalInputFileExtension)) - .collect(Collectors.toList()); + // Check if there are matching files + if (!matchingFiles.isEmpty()) { + // Create a new MultiValueMap for the request body + MultiValueMap body = new LinkedMultiValueMap<>(); - // Check if there are matching files - if (!matchingFiles.isEmpty()) { - // Create a new MultiValueMap for the request body - MultiValueMap body = new LinkedMultiValueMap<>(); + // Add all matching files to the body + for (Resource file : matchingFiles) { + body.add("fileInput", file); + } - // Add all matching files to the body - for (Resource file : matchingFiles) { - body.add("fileInput", file); - } + for (Entry entry : parameters.entrySet()) { + body.add(entry.getKey(), entry.getValue()); + } - for(Entry entry : parameters.entrySet()) { - body.add(entry.getKey(), entry.getValue()); - } - - ResponseEntity response = sendWebRequest(url, body); + ResponseEntity response = sendWebRequest(url, body); - // Handle the response - if (response.getStatusCode().equals(HttpStatus.OK)) { - processOutputFiles(operation, matchingFiles.get(0).getFilename(), response, newOutputFiles); - } else { - // Log error if the response status is not OK - logPrintStream.println("Error in multi-input operation: " + response.getBody()); - hasErrors = true; - } - } else { - logPrintStream.println("No files with extension " + inputFileExtension + " found for multi-input operation " + operation); - hasErrors = true; - } - } - logPrintStream.close(); - outputFiles = newOutputFiles; + // Handle the response + if (response.getStatusCode().equals(HttpStatus.OK)) { + processOutputFiles( + operation, + matchingFiles.get(0).getFilename(), + response, + newOutputFiles); + } else { + // Log error if the response status is not OK + logPrintStream.println( + "Error in multi-input operation: " + response.getBody()); + hasErrors = true; + } + } else { + logPrintStream.println( + "No files with extension " + + inputFileExtension + + " found for multi-input operation " + + operation); + hasErrors = true; + } + } + logPrintStream.close(); + outputFiles = newOutputFiles; + } + if (hasErrors) { + logger.error("Errors occurred during processing. Log: {}", logStream.toString()); + } - } - if (hasErrors) { - logger.error("Errors occurred during processing. Log: {}", logStream.toString()); - } - - return outputFiles; - } + return outputFiles; + } - private ResponseEntity sendWebRequest(String url, MultiValueMap body ){ - RestTemplate restTemplate = new RestTemplate(); - - // Set up headers, including API key + private ResponseEntity sendWebRequest(String url, MultiValueMap body) { + RestTemplate restTemplate = new RestTemplate(); + + // Set up headers, including API key HttpHeaders headers = new HttpHeaders(); String apiKey = getApiKeyForUser(); diff --git a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java index ecdbe22d..483053da 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java @@ -33,67 +33,66 @@ import io.swagger.v3.oas.annotations.tags.Tag; @Tag(name = "General", description = "General APIs") public class GeneralWebController { - - @GetMapping("/pipeline") - @Hidden - public String pipelineForm(Model model) { - model.addAttribute("currentPage", "pipeline"); + @GetMapping("/pipeline") + @Hidden + public String pipelineForm(Model model) { + model.addAttribute("currentPage", "pipeline"); - List pipelineConfigs = new ArrayList<>(); - List> pipelineConfigsWithNames = new ArrayList<>(); - - if(new File("./pipeline/defaultWebUIConfigs/").exists()) { - try (Stream paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) { - List jsonFiles = paths - .filter(Files::isRegularFile) - .filter(p -> p.toString().endsWith(".json")) - .collect(Collectors.toList()); - - for (Path jsonFile : jsonFiles) { - String content = Files.readString(jsonFile, StandardCharsets.UTF_8); - pipelineConfigs.add(content); - } - - for (String config : pipelineConfigs) { - Map jsonContent = new ObjectMapper().readValue(config, new TypeReference>(){}); - - String name = (String) jsonContent.get("name"); - if(name == null || name.length() < 1) { - String filename = jsonFiles.get(pipelineConfigs.indexOf(config)).getFileName().toString(); - name = filename.substring(0, filename.lastIndexOf('.')); - } - Map configWithName = new HashMap<>(); - System.out.println("json" + config); - - System.out.println("name" + name); - configWithName.put("json", config); - configWithName.put("name", name); - pipelineConfigsWithNames.add(configWithName); - } - - - - - } catch (IOException e) { - e.printStackTrace(); - } - } - if(pipelineConfigsWithNames.size() == 0) { - Map configWithName = new HashMap<>(); - configWithName.put("json", ""); - configWithName.put("name", "No preloaded configs found"); - pipelineConfigsWithNames.add(configWithName); - } - model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames); + List pipelineConfigs = new ArrayList<>(); + List> pipelineConfigsWithNames = new ArrayList<>(); - model.addAttribute("pipelineConfigs", pipelineConfigs); + if (new File("./pipeline/defaultWebUIConfigs/").exists()) { + try (Stream paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) { + List jsonFiles = + paths.filter(Files::isRegularFile) + .filter(p -> p.toString().endsWith(".json")) + .collect(Collectors.toList()); - return "pipeline"; - } + for (Path jsonFile : jsonFiles) { + String content = Files.readString(jsonFile, StandardCharsets.UTF_8); + pipelineConfigs.add(content); + } + + for (String config : pipelineConfigs) { + Map jsonContent = + new ObjectMapper() + .readValue(config, new TypeReference>() {}); + + String name = (String) jsonContent.get("name"); + if (name == null || name.length() < 1) { + String filename = + jsonFiles + .get(pipelineConfigs.indexOf(config)) + .getFileName() + .toString(); + name = filename.substring(0, filename.lastIndexOf('.')); + } + Map configWithName = new HashMap<>(); + System.out.println("json" + config); + + System.out.println("name" + name); + configWithName.put("json", config); + configWithName.put("name", name); + pipelineConfigsWithNames.add(configWithName); + } + + } catch (IOException e) { + e.printStackTrace(); + } + } + if (pipelineConfigsWithNames.size() == 0) { + Map configWithName = new HashMap<>(); + configWithName.put("json", ""); + configWithName.put("name", "No preloaded configs found"); + pipelineConfigsWithNames.add(configWithName); + } + model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames); + + model.addAttribute("pipelineConfigs", pipelineConfigs); + + return "pipeline"; + } - - - @GetMapping("/merge-pdfs") @Hidden public String mergePdfForm(Model model) { diff --git a/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java index e52547cf..21324a9c 100644 --- a/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java +++ b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java @@ -2,15 +2,14 @@ package stirling.software.SPDF.utils; public class RequestUriUtils { - public static boolean isStaticResource(String requestURI) { - - return requestURI.startsWith("/css/") - || requestURI.startsWith("/js/") - || requestURI.startsWith("/images/") - || requestURI.startsWith("/public/") - || requestURI.startsWith("/pdfjs/") - || requestURI.endsWith(".svg") - || requestURI.startsWith("/api/v1/info/status"); - + public static boolean isStaticResource(String requestURI) { + + return requestURI.startsWith("/css/") + || requestURI.startsWith("/js/") + || requestURI.startsWith("/images/") + || requestURI.startsWith("/public/") + || requestURI.startsWith("/pdfjs/") + || requestURI.endsWith(".svg") + || requestURI.startsWith("/api/v1/info/status"); } } From 328e87334453da35f1cc25b53c5def36f73a8f32 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 31 Dec 2023 13:35:58 +0000 Subject: [PATCH 09/23] remove prints --- .../software/SPDF/controller/web/GeneralWebController.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java index 483053da..5615a3a6 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java @@ -68,9 +68,6 @@ public class GeneralWebController { name = filename.substring(0, filename.lastIndexOf('.')); } Map configWithName = new HashMap<>(); - System.out.println("json" + config); - - System.out.println("name" + name); configWithName.put("json", config); configWithName.put("name", name); pipelineConfigsWithNames.add(configWithName); From a5ad9e13fe1edd3d5fcfd53635109ca1b0359990 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 31 Dec 2023 14:56:08 +0000 Subject: [PATCH 10/23] todo --- src/main/resources/templates/pipeline.html | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/templates/pipeline.html b/src/main/resources/templates/pipeline.html index 2cf9417c..a4af72b3 100644 --- a/src/main/resources/templates/pipeline.html +++ b/src/main/resources/templates/pipeline.html @@ -99,6 +99,7 @@
  • Fix operation adding requering settings to be openned and saved instead of saving defaults
  • Translation support
  • +
  • Save to browser/Account
  • offline mode checks and testing
  • Improve operation config settings UI
  • From b74819cf6c6307561bab568c34007944efa5a204 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 31 Dec 2023 16:01:27 +0000 Subject: [PATCH 11/23] lang --- src/main/resources/messages_en_GB.properties | 22 ++++++++++ src/main/resources/templates/pipeline.html | 44 +++++++++++++------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/main/resources/messages_en_GB.properties b/src/main/resources/messages_en_GB.properties index 3a21736d..e44aeaf4 100644 --- a/src/main/resources/messages_en_GB.properties +++ b/src/main/resources/messages_en_GB.properties @@ -42,6 +42,7 @@ red=Red green=Green blue=Blue custom=Custom... +WorkInProgess=Work in progress, May not work or be buggy, Please report any ploblems! changedCredsMessage=Credentials changed! notAuthenticatedMessage=User not authenticated. @@ -50,6 +51,27 @@ incorrectPasswordMessage=Current password is incorrect. usernameExistsMessage=New Username already exists. +############### +# Pipeline # +############### +pipeline.header=Pipeline Menu (Alpha) +pipeline.uploadButton=Upload Custom +pipeline.configureButton=Configure +pipeline.defaultOption=Custom +pipeline.submitButton=Submit + +###################### +# Pipeline Options # +###################### +pipelineOptions.header=Pipeline Configuration +pipelineOptions.pipelineNameLabel=Pipeline Name +pipelineOptions.addOperationButton=Add operation +pipelineOptions.pipelineHeader=Pipeline: +pipelineOptions.saveButton=Download +pipelineOptions.validateButton=Validate + + + ############# # NAVBAR # diff --git a/src/main/resources/templates/pipeline.html b/src/main/resources/templates/pipeline.html index a4af72b3..af3581fd 100644 --- a/src/main/resources/templates/pipeline.html +++ b/src/main/resources/templates/pipeline.html @@ -38,13 +38,16 @@
    -

    (Alpha) Pipeline Menu (Huge work in progress, very buggy!)

    +

    +

    - + + data-bs-toggle="modal" data-bs-target="#pipelineSettingsModal" + th:text="#{pipeline.configureButton}"> +
    @@ -52,7 +55,8 @@ +
    - + +
    - + +
    @@ -214,8 +223,11 @@