From 6ca9001fe6115ded1d147051795d870ae1c9c9bf Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 30 Dec 2023 13:42:24 +0000 Subject: [PATCH 01/23] enable status check without apikey --- .../software/SPDF/config/security/SecurityConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java index e0b439db..71394b09 100644 --- a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java +++ b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java @@ -90,7 +90,8 @@ public class SecurityConfiguration { return trimmedUri.startsWith("/login") || trimmedUri.endsWith(".svg") || trimmedUri.startsWith("/register") || trimmedUri.startsWith("/error") || trimmedUri.startsWith("/images/") || trimmedUri.startsWith("/public/") || - trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/"); + trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/") || + trimmedUri.startsWith("api/v1/info/status"); } ).permitAll() .anyRequest().authenticated() From c853465d1d356c96471a0139a38dd7801be79935 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 30 Dec 2023 18:56:07 +0000 Subject: [PATCH 02/23] tests --- .../docker-compose-latest-lite-security.yml | 31 +++++ .../docker-compose-latest-lite.yml | 30 +++++ .../docker-compose-latest-security.yml | 31 +++++ ...ker-compose-latest-ultra-lite-security.yml | 31 +++++ .../docker-compose-latest-ultra-lite.yml | 30 +++++ exampleYmlFiles/docker-compose-latest.yml | 31 +++++ .../security/SecurityConfiguration.java | 2 +- .../security/UserAuthenticationFilter.java | 1 + .../software/SPDF/utils/RequestUriUtils.java | 3 +- test.sh | 122 ++++++++++++++++++ 10 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 exampleYmlFiles/docker-compose-latest-lite-security.yml create mode 100644 exampleYmlFiles/docker-compose-latest-lite.yml create mode 100644 exampleYmlFiles/docker-compose-latest-security.yml create mode 100644 exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml create mode 100644 exampleYmlFiles/docker-compose-latest-ultra-lite.yml create mode 100644 exampleYmlFiles/docker-compose-latest.yml create mode 100644 test.sh diff --git a/exampleYmlFiles/docker-compose-latest-lite-security.yml b/exampleYmlFiles/docker-compose-latest-lite-security.yml new file mode 100644 index 00000000..98b95058 --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-lite-security.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Lite-Security + image: frooodle/s-pdf:latest-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] + interval: 30s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "true" + SECURITY_ENABLELOGIN: "true" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-lite.yml b/exampleYmlFiles/docker-compose-latest-lite.yml new file mode 100644 index 00000000..c2f18efe --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-lite.yml @@ -0,0 +1,30 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Lite + image: frooodle/s-pdf:latest-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] + interval: 20s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "false" + SECURITY_ENABLELOGIN: "false" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-security.yml b/exampleYmlFiles/docker-compose-latest-security.yml new file mode 100644 index 00000000..b36a08a3 --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-security.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Security + image: frooodle/s-pdf:latest-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] + interval: 20s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "true" + SECURITY_ENABLELOGIN: "true" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml new file mode 100644 index 00000000..7b0d6c43 --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Ultra-Lite-Security + image: frooodle/s-pdf:latest-ultra-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] + interval: 30s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "true" + SECURITY_ENABLELOGIN: "true" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml new file mode 100644 index 00000000..5848873d --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml @@ -0,0 +1,30 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF-Ultra-Lite + image: frooodle/s-pdf:latest-ultra-lite-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] + interval: 20s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "false" + SECURITY_ENABLELOGIN: "false" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/exampleYmlFiles/docker-compose-latest.yml b/exampleYmlFiles/docker-compose-latest.yml new file mode 100644 index 00000000..e319695f --- /dev/null +++ b/exampleYmlFiles/docker-compose-latest.yml @@ -0,0 +1,31 @@ +version: '3.3' +services: + stirling-pdf: + container_name: Stirling-PDF + image: frooodle/s-pdf:latest-local + deploy: + resources: + limits: + memory: 1G + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] + interval: 30s + timeout: 10s + retries: 3 + ports: + - 8080:8080 + volumes: + - /stirling/latest/data:/usr/share/tesseract-ocr/5/tessdata:rw + - /stirling/latest/config:/configs:rw + - /stirling/latest/logs:/logs:rw + environment: + DOCKER_ENABLE_SECURITY: "false" + SECURITY_ENABLELOGIN: "false" + SYSTEM_DEFAULTLOCALE: en_US + UI_APPNAME: Stirling-PDF + UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest + UI_APPNAMENAVBAR: Stirling-PDF Latest + SYSTEM_MAXFILESIZE: "100" + METRICS_ENABLED: "true" + SYSTEM_GOOGLEVISIBILITY: "true" + restart: on-failure:5 diff --git a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java index 71394b09..18fad46b 100644 --- a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java +++ b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java @@ -91,7 +91,7 @@ public class SecurityConfiguration { trimmedUri.startsWith("/register") || trimmedUri.startsWith("/error") || trimmedUri.startsWith("/images/") || trimmedUri.startsWith("/public/") || trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/") || - trimmedUri.startsWith("api/v1/info/status"); + trimmedUri.startsWith("/api/v1/info/status"); } ).permitAll() .anyRequest().authenticated() diff --git a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java index ce77e5a4..2b98a9e5 100644 --- a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java +++ b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java @@ -102,6 +102,7 @@ public class UserAuthenticationFilter extends OncePerRequestFilter { contextPath + "/css/", contextPath + "/js/", contextPath + "/pdfjs/", + contextPath + "/api/v1/info/status", contextPath + "/site.webmanifest" }; diff --git a/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java index 0046ee9f..6f07b573 100644 --- a/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java +++ b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java @@ -9,7 +9,8 @@ public class RequestUriUtils { || requestURI.startsWith("/images/") || requestURI.startsWith("/public/") || requestURI.startsWith("/pdfjs/") - || requestURI.endsWith(".svg"); + || requestURI.endsWith(".svg") + || requestURI.startsWith("/api/v1/info/status"); } diff --git a/test.sh b/test.sh new file mode 100644 index 00000000..3b84ef42 --- /dev/null +++ b/test.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Function to check the health of the service with a timeout of 80 seconds +check_health() { + local service_name=$1 + local compose_file=$2 + local end=$((SECONDS+60)) + + echo "Waiting for $service_name to become healthy..." + until [ "$(docker inspect --format='{{json .State.Health.Status}}' "$service_name")" == '"healthy"' ] || [ $SECONDS -ge $end ]; do + sleep 10 + echo "Waiting..." + if [ $SECONDS -ge $end ]; then + echo "$service_name health check timed out after 80 seconds." + return 1 + fi + done + echo "$service_name is healthy!" + return 0 +} + +# Function to test a Docker Compose configuration +test_compose() { + local compose_file=$1 + local service_name=$2 + local status=0 + + echo "Testing $compose_file configuration..." + + # Start up the Docker Compose service + docker-compose -f "$compose_file" up -d + + # Wait for the service to become healthy + if check_health "$service_name" "$compose_file"; then + echo "$service_name test passed." + else + echo "$service_name test failed." + status=1 + fi + + # Perform additional tests if needed + + # Tear down the service + docker-compose -f "$compose_file" down + + return $status +} + +# Keep track of which tests passed and failed +declare -a passed_tests +declare -a failed_tests + +run_tests() { + local test_name=$1 + local compose_file=$2 + + if test_compose "$compose_file" "$test_name"; then + passed_tests+=("$test_name") + else + failed_tests+=("$test_name") + fi +} + +# Main testing routine +main() { + export DOCKER_ENABLE_SECURITY=false + ./gradlew clean build + + # Building Docker images + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-local -f ./Dockerfile . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-lite-local -f ./Dockerfile-lite . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-ultra-lite-local -f ./Dockerfile-ultra-lite . + + # Test each configuration + run_tests "Stirling-PDF-Ultra-Lite" "./exampleYmlFiles/docker-compose-latest-ultra-lite.yml" + run_tests "Stirling-PDF-Lite" "./exampleYmlFiles/docker-compose-latest-lite.yml" + run_tests "Stirling-PDF" "./exampleYmlFiles/docker-compose-latest.yml" + + export DOCKER_ENABLE_SECURITY=true + ./gradlew clean build + + # Building Docker images with security enabled + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-local -f ./Dockerfile . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-lite-local -f ./Dockerfile-lite . + docker build --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-ultra-lite-local -f ./Dockerfile-ultra-lite . + + # Test each configuration with security + run_tests "Stirling-PDF-Ultra-Lite-Security" "./exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml" + run_tests "Stirling-PDF-Lite-Security" "./exampleYmlFiles/docker-compose-latest-lite-security.yml" + run_tests "Stirling-PDF-Security" "./exampleYmlFiles/docker-compose-latest-security.yml" + + # Report results + echo "All tests completed." + + + + if [ ${#passed_tests[@]} -ne 0 ]; then + echo "Passed tests:" + fi + for test in "${passed_tests[@]}"; do + echo -e "\e[32m$test\e[0m" # Green color for passed tests + done + + if [ ${#failed_tests[@]} -ne 0 ]; then + echo "Failed tests:" + fi + for test in "${failed_tests[@]}"; do + echo -e "\e[31m$test\e[0m" # Red color for failed tests + done + + # Check if there are any failed tests and exit with an error code if so + if [ ${#failed_tests[@]} -ne 0 ]; then + echo "Some tests failed." + exit 1 + else + echo "All tests passed successfully." + exit 0 + fi + +} + +main From eda91cc55610ed9731351e2e2a6a1c42f0199bab Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 30 Dec 2023 21:32:04 +0000 Subject: [PATCH 03/23] tests --- .../docker-compose-latest-lite-security.yml | 4 +-- .../docker-compose-latest-lite.yml | 4 +-- .../docker-compose-latest-security.yml | 4 +-- ...ker-compose-latest-ultra-lite-security.yml | 4 +-- .../docker-compose-latest-ultra-lite.yml | 4 +-- exampleYmlFiles/docker-compose-latest.yml | 4 +-- test.sh | 28 +++++++++++-------- 7 files changed, 28 insertions(+), 24 deletions(-) diff --git a/exampleYmlFiles/docker-compose-latest-lite-security.yml b/exampleYmlFiles/docker-compose-latest-lite-security.yml index 98b95058..cb82fdb0 100644 --- a/exampleYmlFiles/docker-compose-latest-lite-security.yml +++ b/exampleYmlFiles/docker-compose-latest-lite-security.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] - interval: 30s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-lite.yml b/exampleYmlFiles/docker-compose-latest-lite.yml index c2f18efe..920ae8d8 100644 --- a/exampleYmlFiles/docker-compose-latest-lite.yml +++ b/exampleYmlFiles/docker-compose-latest-lite.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] - interval: 20s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-security.yml b/exampleYmlFiles/docker-compose-latest-security.yml index b36a08a3..a480c27d 100644 --- a/exampleYmlFiles/docker-compose-latest-security.yml +++ b/exampleYmlFiles/docker-compose-latest-security.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] - interval: 20s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml index 7b0d6c43..a8de634f 100644 --- a/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite-security.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -q 'Please sign in'"] - interval: 30s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest-ultra-lite.yml b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml index 5848873d..b3ad7b00 100644 --- a/exampleYmlFiles/docker-compose-latest-ultra-lite.yml +++ b/exampleYmlFiles/docker-compose-latest-ultra-lite.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] - interval: 20s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/exampleYmlFiles/docker-compose-latest.yml b/exampleYmlFiles/docker-compose-latest.yml index e319695f..21fff579 100644 --- a/exampleYmlFiles/docker-compose-latest.yml +++ b/exampleYmlFiles/docker-compose-latest.yml @@ -9,9 +9,9 @@ services: memory: 1G healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP' && curl -fL http://localhost:8080/ | grep -qv 'Please sign in'"] - interval: 30s + interval: 5s timeout: 10s - retries: 3 + retries: 16 ports: - 8080:8080 volumes: diff --git a/test.sh b/test.sh index 3b84ef42..9617ba14 100644 --- a/test.sh +++ b/test.sh @@ -6,16 +6,17 @@ check_health() { local compose_file=$2 local end=$((SECONDS+60)) - echo "Waiting for $service_name to become healthy..." - until [ "$(docker inspect --format='{{json .State.Health.Status}}' "$service_name")" == '"healthy"' ] || [ $SECONDS -ge $end ]; do - sleep 10 - echo "Waiting..." - if [ $SECONDS -ge $end ]; then - echo "$service_name health check timed out after 80 seconds." - return 1 - fi - done - echo "$service_name is healthy!" + echo -n "Waiting for $service_name to become healthy..." + until [ "$(docker inspect --format='{{json .State.Health.Status}}' "$service_name")" == '"healthy"' ] || [ $SECONDS -ge $end ]; do + sleep 3 + echo -n "." + if [ $SECONDS -ge $end ]; then + echo -e "\n$service_name health check timed out after 80 seconds." + return 1 + fi + done + echo -e "\n$service_name is healthy!" + return 0 } @@ -63,6 +64,8 @@ run_tests() { # Main testing routine main() { + SECONDS=0 + export DOCKER_ENABLE_SECURITY=false ./gradlew clean build @@ -90,8 +93,7 @@ main() { run_tests "Stirling-PDF-Security" "./exampleYmlFiles/docker-compose-latest-security.yml" # Report results - echo "All tests completed." - + echo "All tests completed in $SECONDS seconds." if [ ${#passed_tests[@]} -ne 0 ]; then @@ -108,6 +110,8 @@ main() { echo -e "\e[31m$test\e[0m" # Red color for failed tests done + + # Check if there are any failed tests and exit with an error code if so if [ ${#failed_tests[@]} -ne 0 ]; then echo "Some tests failed." From f535387ac4dc943eb0675a1ca02849b243e9fe6b Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 31 Dec 2023 13:05:38 +0000 Subject: [PATCH 04/23] pipeline enhance for MI --- .../api/pipeline/PipelineProcessor.java | 4 +- src/main/resources/static/js/pipeline.js | 997 +++++++++--------- src/main/resources/templates/pipeline.html | 2 + 3 files changed, 521 insertions(+), 482 deletions(-) diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java index 8b4b2ef4..c2841a05 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java @@ -138,7 +138,7 @@ public class PipelineProcessor { hasErrors = true; } - outputFiles = newOutputFiles; + } } else { @@ -177,11 +177,13 @@ public class PipelineProcessor { } } logPrintStream.close(); + outputFiles = newOutputFiles; } if (hasErrors) { logger.error("Errors occurred during processing. Log: {}", logStream.toString()); } + return outputFiles; } diff --git a/src/main/resources/static/js/pipeline.js b/src/main/resources/static/js/pipeline.js index 4fcde3a0..8db09c48 100644 --- a/src/main/resources/static/js/pipeline.js +++ b/src/main/resources/static/js/pipeline.js @@ -1,490 +1,511 @@ -document.getElementById('validateButton').addEventListener('click', function(event) { - event.preventDefault(); - validatePipeline(); -}); -function validatePipeline() { - let pipelineListItems = document.getElementById('pipelineList').children; - let isValid = true; - let containsAddPassword = false; - for (let i = 0; i < pipelineListItems.length - 1; i++) { - let currentOperation = pipelineListItems[i].querySelector('.operationName').textContent; - let nextOperation = pipelineListItems[i + 1].querySelector('.operationName').textContent; - if (currentOperation === '/add-password') { - containsAddPassword = true; - } - - let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; - let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; - - // Strip off 'ZIP-' prefix - currentOperationDescription = currentOperationDescription.replace("ZIP-", ''); - nextOperationDescription = nextOperationDescription.replace("ZIP-", ''); - - let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; - let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; - - // Splitting in case of multiple possible output/input - let currentOperationOutputArr = currentOperationOutput.split('/'); - let nextOperationInputArr = nextOperationInput.split('/'); - - if (currentOperationOutput !== 'ANY' && nextOperationInput !== 'ANY') { - let intersection = currentOperationOutputArr.filter(value => nextOperationInputArr.includes(value)); - console.log(`Intersection: ${intersection}`); - - if (intersection.length === 0) { - updateValidateButton(false); - isValid = false; - console.log(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); - alert(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); - break; - } - } - } - if (containsAddPassword && pipelineListItems[pipelineListItems.length - 1].querySelector('.operationName').textContent !== '/add-password') { - updateValidateButton(false); - alert('The "add-password" operation should be at the end of the operations sequence. Please adjust the operations order.'); - return false; - } - if (isValid) { - console.log('Pipeline is valid'); - // Continue with the pipeline operation - } else { - console.error('Pipeline is not valid'); - // Stop operation, maybe display an error to the user - } - updateValidateButton(isValid); - return isValid; -} - -function updateValidateButton(isValid) { - var validateButton = document.getElementById('validateButton'); - if (isValid) { - validateButton.classList.remove('btn-danger'); - validateButton.classList.add('btn-success'); - } else { - validateButton.classList.remove('btn-success'); - validateButton.classList.add('btn-danger'); - } -} - - - - -document.getElementById('submitConfigBtn').addEventListener('click', function() { - - if (validatePipeline() === false) { - return; - } - let selectedOperation = document.getElementById('operationsDropdown').value; - - - - var pipelineName = document.getElementById('pipelineName').value; - let pipelineList = document.getElementById('pipelineList').children; - let pipelineConfig = { - "name": pipelineName, - "pipeline": [], - "_examples": { - "outputDir": "{outputFolder}/{folderName}", - "outputFileName": "{filename}-{pipelineName}-{date}-{time}" - }, - "outputDir": "httpWebRequest", - "outputFileName": "{filename}" - }; - - for (let i = 0; i < pipelineList.length; i++) { - let operationName = pipelineList[i].querySelector('.operationName').textContent; - let parameters = operationSettings[operationName] || {}; - - pipelineConfig.pipeline.push({ - "operation": operationName, - "parameters": parameters - }); - } - - - - - - - - - - - - - - let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2); - - let formData = new FormData(); - - let fileInput = document.getElementById('fileInput-input'); - let files = fileInput.files; - - for (let i = 0; i < files.length; i++) { - console.log("files[i]", files[i].name); - formData.append('fileInput', files[i], files[i].name); - } - - console.log("pipelineConfigJson", pipelineConfigJson); - formData.append('json', pipelineConfigJson); - console.log("formData", formData); - - fetch('api/v1/pipeline/handleData', { - method: 'POST', - body: formData - }) - .then(response => { - // Save the response to use it later - const responseToUseLater = response; - - return response.blob().then(blob => { - let url = window.URL.createObjectURL(blob); - let a = document.createElement('a'); - a.href = url; - - // Use responseToUseLater instead of response - const contentDisposition = responseToUseLater.headers.get('Content-Disposition'); - let filename = 'download'; - if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) { - filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim(); - } - a.download = filename; - - document.body.appendChild(a); - a.click(); - a.remove(); - }); - }) - .catch((error) => { - console.error('Error:', error); + document.getElementById('validateButton').addEventListener('click', function(event) { + event.preventDefault(); + validatePipeline(); }); - -}); - -let apiDocs = {}; -let apiSchemas = {}; -let operationSettings = {}; - -fetch('v1/api-docs') - .then(response => response.json()) - .then(data => { - - apiDocs = data.paths; - apiSchemas = data.components.schemas; - let operationsDropdown = document.getElementById('operationsDropdown'); - const ignoreOperations = ["/api/v1/pipeline/handleData", "/api/v1/pipeline/operationToIgnore"]; // Add the operations you want to ignore here - - operationsDropdown.innerHTML = ''; - - let operationsByTag = {}; - - // Group operations by tags - Object.keys(data.paths).forEach(operationPath => { - let operation = data.paths[operationPath].post; - if(!operation || !operation.description) { - console.log(operationPath); + function validatePipeline() { + let pipelineListItems = document.getElementById('pipelineList').children; + let isValid = true; + let containsAddPassword = false; + for (let i = 0; i < pipelineListItems.length - 1; i++) { + let currentOperation = pipelineListItems[i].querySelector('.operationName').textContent; + let nextOperation = pipelineListItems[i + 1].querySelector('.operationName').textContent; + if (currentOperation === '/add-password') { + containsAddPassword = true; } - if (operation && !ignoreOperations.includes(operationPath) && !operation.description.includes("Type:MISO")) { - let operationTag = operation.tags[0]; // This assumes each operation has exactly one tag - if (!operationsByTag[operationTag]) { - operationsByTag[operationTag] = []; + + let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; + let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; + + // Strip off 'ZIP-' prefix + currentOperationDescription = currentOperationDescription.replace("ZIP-", ''); + nextOperationDescription = nextOperationDescription.replace("ZIP-", ''); + + let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; + let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; + + // Splitting in case of multiple possible output/input + let currentOperationOutputArr = currentOperationOutput.split('/'); + let nextOperationInputArr = nextOperationInput.split('/'); + + if (currentOperationOutput !== 'ANY' && nextOperationInput !== 'ANY') { + let intersection = currentOperationOutputArr.filter(value => nextOperationInputArr.includes(value)); + console.log(`Intersection: ${intersection}`); + + if (intersection.length === 0) { + updateValidateButton(false); + isValid = false; + console.log(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); + alert(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); + break; } - operationsByTag[operationTag].push(operationPath); } - }); - // Specify the order of tags - let tagOrder = ["General", "Security", "Convert", "Misc", "Filter"]; - - // Create dropdown options - tagOrder.forEach(tag => { - if (operationsByTag[tag]) { - let group = document.createElement('optgroup'); - group.label = tag; - - operationsByTag[tag].forEach(operationPath => { - let option = document.createElement('option'); - - let operationPathDisplay = operationPath - operationPathDisplay = operationPath.replace(new RegExp("api/v1/" + tag.toLowerCase() + "/", 'i'), ""); - - - if(operationPath.includes("/convert")){ - operationPathDisplay = operationPathDisplay.replace(/^\//, '').replaceAll("/", " to "); - } else { - operationPathDisplay = operationPathDisplay.replace(/\//g, ''); // Remove slashes - } - operationPathDisplay = operationPathDisplay.replaceAll(" ","-"); - option.textContent = operationPathDisplay; - option.value = operationPath; // Keep the value with slashes for querying - group.appendChild(option); - }); - - operationsDropdown.appendChild(group); - } - }); - }); - - -document.getElementById('addOperationBtn').addEventListener('click', function() { - let selectedOperation = document.getElementById('operationsDropdown').value; - let pipelineList = document.getElementById('pipelineList'); - - let listItem = document.createElement('li'); - listItem.className = "list-group-item"; - let hasSettings = false; - if (apiDocs[selectedOperation] && apiDocs[selectedOperation].post) { - const postMethod = apiDocs[selectedOperation].post; - - // Check if parameters exist - if (postMethod.parameters && postMethod.parameters.length > 0) { - hasSettings = true; - } else if (postMethod.requestBody && postMethod.requestBody.content['multipart/form-data']) { - // Extract the reference key - const refKey = postMethod.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); - // Check if the referenced schema exists and has properties - if (apiSchemas[refKey] && Object.keys(apiSchemas[refKey].properties).length > 0) { - hasSettings = true; - } - } + } + if (containsAddPassword && pipelineListItems[pipelineListItems.length - 1].querySelector('.operationName').textContent !== '/add-password') { + updateValidateButton(false); + alert('The "add-password" operation should be at the end of the operations sequence. Please adjust the operations order.'); + return false; + } + if (isValid) { + console.log('Pipeline is valid'); + // Continue with the pipeline operation + } else { + console.error('Pipeline is not valid'); + // Stop operation, maybe display an error to the user + } + updateValidateButton(isValid); + return isValid; } - - - - - listItem.innerHTML = ` -