diff --git a/src/main/java/stirling/software/SPDF/SPdfApplication.java b/src/main/java/stirling/software/SPDF/SPdfApplication.java index f4c8c7c5..76446258 100644 --- a/src/main/java/stirling/software/SPDF/SPdfApplication.java +++ b/src/main/java/stirling/software/SPDF/SPdfApplication.java @@ -1,73 +1,76 @@ -package stirling.software.SPDF; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.core.env.Environment; -import org.springframework.scheduling.annotation.EnableScheduling; - -import jakarta.annotation.PostConstruct; -import stirling.software.SPDF.utils.GeneralUtils; - -@SpringBootApplication -@EnableScheduling -public class SPdfApplication { - - @Autowired - private Environment env; - - @PostConstruct - public void init() { - // Check if the BROWSER_OPEN environment variable is set to true - String browserOpenEnv = env.getProperty("BROWSER_OPEN"); - boolean browserOpen = browserOpenEnv != null && browserOpenEnv.equalsIgnoreCase("true"); - - if (browserOpen) { - try { - String port = env.getProperty("local.server.port"); - if(port == null || port.length() == 0) { - port="8080"; - } - String url = "http://localhost:" + port; - - String os = System.getProperty("os.name").toLowerCase(); - Runtime rt = Runtime.getRuntime(); - if (os.contains("win")) { - // For Windows - rt.exec("rundll32 url.dll,FileProtocolHandler " + url); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - public static void main(String[] args) { - SpringApplication.run(SPdfApplication.class, args); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - GeneralUtils.createDir("customFiles/static/"); - GeneralUtils.createDir("customFiles/templates/"); - GeneralUtils.createDir("config"); - System.out.println("Stirling-PDF Started."); - - String port = System.getProperty("local.server.port"); - if(port == null || port.length() == 0) { - port="8080"; - } - String url = "http://localhost:" + port; - System.out.println("Navigate to " + url); - } - - +package stirling.software.SPDF; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.annotation.EnableScheduling; + +import jakarta.annotation.PostConstruct; +import stirling.software.SPDF.utils.GeneralUtils; + +@SpringBootApplication +@EnableScheduling +public class SPdfApplication { + + @Autowired + private Environment env; + + @PostConstruct + public void init() { + // Check if the BROWSER_OPEN environment variable is set to true + String browserOpenEnv = env.getProperty("BROWSER_OPEN"); + boolean browserOpen = browserOpenEnv != null && browserOpenEnv.equalsIgnoreCase("true"); + + if (browserOpen) { + try { + String port = env.getProperty("local.server.port"); + if(port == null || port.length() == 0) { + port="8080"; + } + String url = "http://localhost:" + port; + + String os = System.getProperty("os.name").toLowerCase(); + Runtime rt = Runtime.getRuntime(); + if (os.contains("win")) { + // For Windows + rt.exec("rundll32 url.dll,FileProtocolHandler " + url); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + public static void main(String[] args) { + SpringApplication.run(SPdfApplication.class, args); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + GeneralUtils.createDir("customFiles/static/"); + GeneralUtils.createDir("customFiles/templates/"); + GeneralUtils.createDir("config"); + + + + System.out.println("Stirling-PDF Started."); + + String port = System.getProperty("local.server.port"); + if(port == null || port.length() == 0) { + port="8080"; + } + String url = "http://localhost:" + port; + System.out.println("Navigate to " + url); + } + + } \ No newline at end of file diff --git a/src/main/java/stirling/software/SPDF/config/Beans.java b/src/main/java/stirling/software/SPDF/config/Beans.java index 7a963ba4..f8ff302e 100644 --- a/src/main/java/stirling/software/SPDF/config/Beans.java +++ b/src/main/java/stirling/software/SPDF/config/Beans.java @@ -1,60 +1,60 @@ -package stirling.software.SPDF.config; - -import java.util.Locale; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.LocaleResolver; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import org.springframework.web.servlet.i18n.LocaleChangeInterceptor; -import org.springframework.web.servlet.i18n.SessionLocaleResolver; - -@Configuration -public class Beans implements WebMvcConfigurer { - - @Override - public void addInterceptors(InterceptorRegistry registry) { - registry.addInterceptor(localeChangeInterceptor()); - registry.addInterceptor(new CleanUrlInterceptor()); - } - - @Bean - public LocaleChangeInterceptor localeChangeInterceptor() { - LocaleChangeInterceptor lci = new LocaleChangeInterceptor(); - lci.setParamName("lang"); - return lci; - } - - @Bean - public LocaleResolver localeResolver() { - SessionLocaleResolver slr = new SessionLocaleResolver(); - - String appLocaleEnv = System.getProperty("APP_LOCALE"); - if (appLocaleEnv == null) - appLocaleEnv = System.getenv("APP_LOCALE"); - Locale defaultLocale = Locale.UK; // Fallback to UK locale if environment variable is not set - - if (appLocaleEnv != null && !appLocaleEnv.isEmpty()) { - Locale tempLocale = Locale.forLanguageTag(appLocaleEnv); - String tempLanguageTag = tempLocale.toLanguageTag(); - - if (appLocaleEnv.equalsIgnoreCase(tempLanguageTag)) { - defaultLocale = tempLocale; - } else { - tempLocale = Locale.forLanguageTag(appLocaleEnv.replace("_","-")); - tempLanguageTag = tempLocale.toLanguageTag(); - - if (appLocaleEnv.equalsIgnoreCase(tempLanguageTag)) { - defaultLocale = tempLocale; - } else { - System.err.println("Invalid APP_LOCALE environment variable value. Falling back to default Locale.UK."); - } - } - } - - slr.setDefaultLocale(defaultLocale); - return slr; - } - -} +package stirling.software.SPDF.config; + +import java.util.Locale; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.LocaleResolver; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import org.springframework.web.servlet.i18n.LocaleChangeInterceptor; +import org.springframework.web.servlet.i18n.SessionLocaleResolver; + +@Configuration +public class Beans implements WebMvcConfigurer { + + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(localeChangeInterceptor()); + registry.addInterceptor(new CleanUrlInterceptor()); + } + + @Bean + public LocaleChangeInterceptor localeChangeInterceptor() { + LocaleChangeInterceptor lci = new LocaleChangeInterceptor(); + lci.setParamName("lang"); + return lci; + } + + @Bean + public LocaleResolver localeResolver() { + SessionLocaleResolver slr = new SessionLocaleResolver(); + + String appLocaleEnv = System.getProperty("APP_LOCALE"); + if (appLocaleEnv == null) + appLocaleEnv = System.getenv("APP_LOCALE"); + Locale defaultLocale = Locale.UK; // Fallback to UK locale if environment variable is not set + + if (appLocaleEnv != null && !appLocaleEnv.isEmpty()) { + Locale tempLocale = Locale.forLanguageTag(appLocaleEnv); + String tempLanguageTag = tempLocale.toLanguageTag(); + + if (appLocaleEnv.equalsIgnoreCase(tempLanguageTag)) { + defaultLocale = tempLocale; + } else { + tempLocale = Locale.forLanguageTag(appLocaleEnv.replace("_","-")); + tempLanguageTag = tempLocale.toLanguageTag(); + + if (appLocaleEnv.equalsIgnoreCase(tempLanguageTag)) { + defaultLocale = tempLocale; + } else { + System.err.println("Invalid APP_LOCALE environment variable value. Falling back to default Locale.UK."); + } + } + } + + slr.setDefaultLocale(defaultLocale); + return slr; + } + +} diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/Controller.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/Controller.java index 1c32c2ef..8193e0c8 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/Controller.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/Controller.java @@ -1,497 +1,497 @@ -package stirling.software.SPDF.controller.api.pipeline; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; -import java.io.FileOutputStream; -import java.io.OutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.core.io.Resource; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RequestPart; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.multipart.MultipartFile; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import io.swagger.v3.oas.annotations.tags.Tag; -import stirling.software.SPDF.model.PipelineConfig; -import stirling.software.SPDF.model.PipelineOperation; -import stirling.software.SPDF.utils.WebResponseUtils; - -@RestController -@Tag(name = "Pipeline", description = "Pipeline APIs") -public class Controller { - - private static final Logger logger = LoggerFactory.getLogger(Controller.class); - @Autowired - private ObjectMapper objectMapper; - - final String jsonFileName = "pipelineConfig.json"; - final String watchedFoldersDir = "watchedFolders/"; - final String finishedFoldersDir = "finishedFolders/"; - - @Scheduled(fixedRate = 25000) - public void scanFolders() { - logger.info("Scanning folders..."); - Path watchedFolderPath = Paths.get(watchedFoldersDir); - if (!Files.exists(watchedFolderPath)) { - try { - Files.createDirectories(watchedFolderPath); - logger.info("Created directory: {}", watchedFolderPath); - } catch (IOException e) { - logger.error("Error creating directory: {}", watchedFolderPath, e); - return; - } - } - try (Stream paths = Files.walk(watchedFolderPath)) { - paths.filter(Files::isDirectory).forEach(t -> { - try { - if (!t.equals(watchedFolderPath) && !t.endsWith("processing")) { - handleDirectory(t); - } - } catch (Exception e) { - logger.error("Error handling directory: {}", t, e); - } - }); - } catch (Exception e) { - logger.error("Error walking through directory: {}", watchedFolderPath, e); - } - } - - private void handleDirectory(Path dir) throws Exception { - logger.info("Handling directory: {}", dir); - Path jsonFile = dir.resolve(jsonFileName); - Path processingDir = dir.resolve("processing"); // Directory to move files during processing - if (!Files.exists(processingDir)) { - Files.createDirectory(processingDir); - logger.info("Created processing directory: {}", processingDir); - } - - if (Files.exists(jsonFile)) { - // Read JSON file - String jsonString; - try { - jsonString = new String(Files.readAllBytes(jsonFile)); - logger.info("Read JSON file: {}", jsonFile); - } catch (IOException e) { - logger.error("Error reading JSON file: {}", jsonFile, e); - return; - } - - // Decode JSON to PipelineConfig - PipelineConfig config; - try { - config = objectMapper.readValue(jsonString, PipelineConfig.class); - // Assuming your PipelineConfig class has getters for all necessary fields, you - // can perform checks here - if (config.getOperations() == null || config.getOutputDir() == null || config.getName() == null) { - throw new IOException("Invalid JSON format"); - } - } catch (IOException e) { - logger.error("Error parsing PipelineConfig: {}", jsonString, e); - return; - } - - // For each operation in the pipeline - for (PipelineOperation operation : config.getOperations()) { - // Collect all files based on fileInput - File[] files; - String fileInput = (String) operation.getParameters().get("fileInput"); - if ("automated".equals(fileInput)) { - // If fileInput is "automated", process all files in the directory - try (Stream paths = Files.list(dir)) { - files = paths - .filter(path -> !Files.isDirectory(path)) // exclude directories - .filter(path -> !path.equals(jsonFile)) // exclude jsonFile - .map(Path::toFile) - .toArray(File[]::new); - - } catch (IOException e) { - e.printStackTrace(); - return; - } - } else { - // If fileInput contains a path, process only this file - files = new File[] { new File(fileInput) }; - } - - // Prepare the files for processing - List filesToProcess = new ArrayList<>(); - for (File file : files) { - logger.info(file.getName()); - logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName())); - Files.move(file.toPath(), processingDir.resolve(file.getName())); - filesToProcess.add(processingDir.resolve(file.getName()).toFile()); - } - - // Process the files - try { - List resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString); - - if(resources == null) { - return; - } - // Move resultant files and rename them as per config in JSON file - for (Resource resource : resources) { - String resourceName = resource.getFilename(); - String baseName = resourceName.substring(0, resourceName.lastIndexOf(".")); - String extension = resourceName.substring(resourceName.lastIndexOf(".")+1); - - String outputFileName = config.getOutputPattern().replace("{filename}", baseName); - - outputFileName = outputFileName.replace("{pipelineName}", config.getName()); - DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - outputFileName = outputFileName.replace("{date}", LocalDate.now().format(dateFormatter)); - DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HHmmss"); - outputFileName = outputFileName.replace("{time}", LocalTime.now().format(timeFormatter)); - - outputFileName += "." + extension; - // {filename} {folder} {date} {tmime} {pipeline} - String outputDir = config.getOutputDir(); - - // Check if the environment variable 'automatedOutputFolder' is set - String outputFolder = System.getenv("automatedOutputFolder"); - - if (outputFolder == null || outputFolder.isEmpty()) { - // If the environment variable is not set, use the default value - outputFolder = finishedFoldersDir; - } - logger.info("outputDir 0={}", outputDir); - // Replace the placeholders in the outputDir string - outputDir = outputDir.replace("{outputFolder}", outputFolder); - outputDir = outputDir.replace("{folderName}", dir.toString()); - logger.info("outputDir 1={}", outputDir); - outputDir = outputDir.replace("\\watchedFolders", ""); - outputDir = outputDir.replace("//watchedFolders", ""); - outputDir = outputDir.replace("\\\\watchedFolders", ""); - outputDir = outputDir.replace("/watchedFolders", ""); - - Path outputPath; - logger.info("outputDir 2={}", outputDir); - if (Paths.get(outputDir).isAbsolute()) { - // If it's an absolute path, use it directly - outputPath = Paths.get(outputDir); - } else { - // If it's a relative path, make it relative to the current working directory - outputPath = Paths.get(".", outputDir); - } - - logger.info("outputPath={}", outputPath); - - if (!Files.exists(outputPath)) { - try { - Files.createDirectories(outputPath); - logger.info("Created directory: {}", outputPath); - } catch (IOException e) { - logger.error("Error creating directory: {}", outputPath, e); - return; - } - } - logger.info("outputPath {}", outputPath); - logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString()); - File newFile = new File(outputPath.resolve(outputFileName).toString()); - OutputStream os = new FileOutputStream(newFile); - os.write(((ByteArrayResource)resource).getByteArray()); - os.close(); - logger.info("made {}", outputPath.resolve(outputFileName)); - } - - // If successful, delete the original files - for (File file : filesToProcess) { - Files.deleteIfExists(processingDir.resolve(file.getName())); - } - } catch (Exception e) { - // If an error occurs, move the original files back - for (File file : filesToProcess) { - Files.move(processingDir.resolve(file.getName()), file.toPath()); - } - throw e; - } - } - } - } - - List processFiles(List outputFiles, String jsonString) throws Exception { - - logger.info("Processing files... " + outputFiles); - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - - JsonNode pipelineNode = jsonNode.get("pipeline"); - ByteArrayOutputStream logStream = new ByteArrayOutputStream(); - PrintStream logPrintStream = new PrintStream(logStream); - - boolean hasErrors = false; - - for (JsonNode operationNode : pipelineNode) { - String operation = operationNode.get("operation").asText(); - logger.info("Running operation: {}", operation); - JsonNode parametersNode = operationNode.get("parameters"); - String inputFileExtension = ""; - if (operationNode.has("inputFileType")) { - inputFileExtension = operationNode.get("inputFileType").asText(); - } else { - inputFileExtension = ".pdf"; - } - - List newOutputFiles = new ArrayList<>(); - boolean hasInputFileType = false; - - for (Resource file : outputFiles) { - if (file.getFilename().endsWith(inputFileExtension)) { - hasInputFileType = true; - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("fileInput", file); - - Iterator> parameters = parametersNode.fields(); - while (parameters.hasNext()) { - Map.Entry parameter = parameters.next(); - body.add(parameter.getKey(), parameter.getValue().asText()); - } - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - - HttpEntity> entity = new HttpEntity<>(body, headers); - - RestTemplate restTemplate = new RestTemplate(); - String url = "http://localhost:8080/" + operation; - - ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class); - - if (!response.getStatusCode().equals(HttpStatus.OK)) { - logPrintStream.println("Error: " + response.getBody()); - hasErrors = true; - continue; - } - - // Check if the response body is a zip file - if (isZip(response.getBody())) { - // Unzip the file and add all the files to the new output files - newOutputFiles.addAll(unzip(response.getBody())); - } else { - Resource outputResource = new ByteArrayResource(response.getBody()) { - @Override - public String getFilename() { - return file.getFilename(); // Preserving original filename - } - }; - newOutputFiles.add(outputResource); - } - } - - if (!hasInputFileType) { - logPrintStream.println( - "No files with extension " + inputFileExtension + " found for operation " + operation); - hasErrors = true; - } - - outputFiles = newOutputFiles; - } - logPrintStream.close(); - - } - if (hasErrors) { - logger.error("Errors occurred during processing. Log: {}", logStream.toString()); - } - return outputFiles; - } - - List handleFiles(File[] files, String jsonString) throws Exception { - if(files == null || files.length == 0) { - logger.info("No files"); - return null; - } - - logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); - - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - - JsonNode pipelineNode = jsonNode.get("pipeline"); - - boolean hasErrors = false; - List outputFiles = new ArrayList<>(); - - for (File file : files) { - Path path = Paths.get(file.getAbsolutePath()); - System.out.println("Reading file: " + path); // debug statement - - if (Files.exists(path)) { - Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) { - @Override - public String getFilename() { - return file.getName(); - } - }; - outputFiles.add(fileResource); - } else { - System.out.println("File not found: " + path); // debug statement - } - } - logger.info("Files successfully loaded. Starting processing..."); - return processFiles(outputFiles, jsonString); - } - - List handleFiles(MultipartFile[] files, String jsonString) throws Exception { - if(files == null || files.length == 0) { - logger.info("No files"); - return null; - } - logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - - JsonNode pipelineNode = jsonNode.get("pipeline"); - - boolean hasErrors = false; - List outputFiles = new ArrayList<>(); - - for (MultipartFile file : files) { - Resource fileResource = new ByteArrayResource(file.getBytes()) { - @Override - public String getFilename() { - return file.getOriginalFilename(); - } - }; - outputFiles.add(fileResource); - } - logger.info("Files successfully loaded. Starting processing..."); - return processFiles(outputFiles, jsonString); - } - - @PostMapping("/handleData") - public ResponseEntity handleData(@RequestPart("fileInput") MultipartFile[] files, - @RequestParam("json") String jsonString) { - logger.info("Received POST request to /handleData with {} files", files.length); - try { - List outputFiles = handleFiles(files, jsonString); - - if (outputFiles != null && outputFiles.size() == 1) { - // If there is only one file, return it directly - Resource singleFile = outputFiles.get(0); - InputStream is = singleFile.getInputStream(); - byte[] bytes = new byte[(int) singleFile.contentLength()]; - is.read(bytes); - is.close(); - - logger.info("Returning single file response..."); - return WebResponseUtils.bytesToWebResponse(bytes, singleFile.getFilename(), - MediaType.APPLICATION_OCTET_STREAM); - } else if (outputFiles == null) { - return null; - } - - // Create a ByteArrayOutputStream to hold the zip - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ZipOutputStream zipOut = new ZipOutputStream(baos); - - // Loop through each file and add it to the zip - for (Resource file : outputFiles) { - ZipEntry zipEntry = new ZipEntry(file.getFilename()); - zipOut.putNextEntry(zipEntry); - - // Read the file into a byte array - InputStream is = file.getInputStream(); - byte[] bytes = new byte[(int) file.contentLength()]; - is.read(bytes); - - // Write the bytes of the file to the zip - zipOut.write(bytes, 0, bytes.length); - zipOut.closeEntry(); - - is.close(); - } - - zipOut.close(); - - logger.info("Returning zipped file response..."); - return WebResponseUtils.boasToWebResponse(baos, "output.zip", MediaType.APPLICATION_OCTET_STREAM); - } catch (Exception e) { - logger.error("Error handling data: ", e); - return null; - } - } - - private boolean isZip(byte[] data) { - if (data == null || data.length < 4) { - return false; - } - - // Check the first four bytes of the data against the standard zip magic number - return data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04; - } - - private List unzip(byte[] data) throws IOException { - logger.info("Unzipping data of length: {}", data.length); - List unzippedFiles = new ArrayList<>(); - - try (ByteArrayInputStream bais = new ByteArrayInputStream(data); - ZipInputStream zis = new ZipInputStream(bais)) { - - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int count; - - while ((count = zis.read(buffer)) != -1) { - baos.write(buffer, 0, count); - } - - final String filename = entry.getName(); - Resource fileResource = new ByteArrayResource(baos.toByteArray()) { - @Override - public String getFilename() { - return filename; - } - }; - - // If the unzipped file is a zip file, unzip it - if (isZip(baos.toByteArray())) { - logger.info("File {} is a zip file. Unzipping...", filename); - unzippedFiles.addAll(unzip(baos.toByteArray())); - } else { - unzippedFiles.add(fileResource); - } - } - } - - logger.info("Unzipping completed. {} files were unzipped.", unzippedFiles.size()); - return unzippedFiles; - } - -} +package stirling.software.SPDF.controller.api.pipeline; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; +import java.io.FileOutputStream; +import java.io.OutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.swagger.v3.oas.annotations.tags.Tag; +import stirling.software.SPDF.model.PipelineConfig; +import stirling.software.SPDF.model.PipelineOperation; +import stirling.software.SPDF.utils.WebResponseUtils; + +@RestController +@Tag(name = "Pipeline", description = "Pipeline APIs") +public class Controller { + + private static final Logger logger = LoggerFactory.getLogger(Controller.class); + @Autowired + private ObjectMapper objectMapper; + + final String jsonFileName = "pipelineConfig.json"; + final String watchedFoldersDir = "./pipeline/watchedFolders/"; + final String finishedFoldersDir = "./pipeline/finishedFolders/"; + + @Scheduled(fixedRate = 25000) + public void scanFolders() { + logger.info("Scanning folders..."); + Path watchedFolderPath = Paths.get(watchedFoldersDir); + if (!Files.exists(watchedFolderPath)) { + try { + Files.createDirectories(watchedFolderPath); + logger.info("Created directory: {}", watchedFolderPath); + } catch (IOException e) { + logger.error("Error creating directory: {}", watchedFolderPath, e); + return; + } + } + try (Stream paths = Files.walk(watchedFolderPath)) { + paths.filter(Files::isDirectory).forEach(t -> { + try { + if (!t.equals(watchedFolderPath) && !t.endsWith("processing")) { + handleDirectory(t); + } + } catch (Exception e) { + logger.error("Error handling directory: {}", t, e); + } + }); + } catch (Exception e) { + logger.error("Error walking through directory: {}", watchedFolderPath, e); + } + } + + private void handleDirectory(Path dir) throws Exception { + logger.info("Handling directory: {}", dir); + Path jsonFile = dir.resolve(jsonFileName); + Path processingDir = dir.resolve("processing"); // Directory to move files during processing + if (!Files.exists(processingDir)) { + Files.createDirectory(processingDir); + logger.info("Created processing directory: {}", processingDir); + } + + if (Files.exists(jsonFile)) { + // Read JSON file + String jsonString; + try { + jsonString = new String(Files.readAllBytes(jsonFile)); + logger.info("Read JSON file: {}", jsonFile); + } catch (IOException e) { + logger.error("Error reading JSON file: {}", jsonFile, e); + return; + } + + // Decode JSON to PipelineConfig + PipelineConfig config; + try { + config = objectMapper.readValue(jsonString, PipelineConfig.class); + // Assuming your PipelineConfig class has getters for all necessary fields, you + // can perform checks here + if (config.getOperations() == null || config.getOutputDir() == null || config.getName() == null) { + throw new IOException("Invalid JSON format"); + } + } catch (IOException e) { + logger.error("Error parsing PipelineConfig: {}", jsonString, e); + return; + } + + // For each operation in the pipeline + for (PipelineOperation operation : config.getOperations()) { + // Collect all files based on fileInput + File[] files; + String fileInput = (String) operation.getParameters().get("fileInput"); + if ("automated".equals(fileInput)) { + // If fileInput is "automated", process all files in the directory + try (Stream paths = Files.list(dir)) { + files = paths + .filter(path -> !Files.isDirectory(path)) // exclude directories + .filter(path -> !path.equals(jsonFile)) // exclude jsonFile + .map(Path::toFile) + .toArray(File[]::new); + + } catch (IOException e) { + e.printStackTrace(); + return; + } + } else { + // If fileInput contains a path, process only this file + files = new File[] { new File(fileInput) }; + } + + // Prepare the files for processing + List filesToProcess = new ArrayList<>(); + for (File file : files) { + logger.info(file.getName()); + logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName())); + Files.move(file.toPath(), processingDir.resolve(file.getName())); + filesToProcess.add(processingDir.resolve(file.getName()).toFile()); + } + + // Process the files + try { + List resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString); + + if(resources == null) { + return; + } + // Move resultant files and rename them as per config in JSON file + for (Resource resource : resources) { + String resourceName = resource.getFilename(); + String baseName = resourceName.substring(0, resourceName.lastIndexOf(".")); + String extension = resourceName.substring(resourceName.lastIndexOf(".")+1); + + String outputFileName = config.getOutputPattern().replace("{filename}", baseName); + + outputFileName = outputFileName.replace("{pipelineName}", config.getName()); + DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + outputFileName = outputFileName.replace("{date}", LocalDate.now().format(dateFormatter)); + DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HHmmss"); + outputFileName = outputFileName.replace("{time}", LocalTime.now().format(timeFormatter)); + + outputFileName += "." + extension; + // {filename} {folder} {date} {tmime} {pipeline} + String outputDir = config.getOutputDir(); + + // Check if the environment variable 'automatedOutputFolder' is set + String outputFolder = System.getenv("automatedOutputFolder"); + + if (outputFolder == null || outputFolder.isEmpty()) { + // If the environment variable is not set, use the default value + outputFolder = finishedFoldersDir; + } + logger.info("outputDir 0={}", outputDir); + // Replace the placeholders in the outputDir string + outputDir = outputDir.replace("{outputFolder}", outputFolder); + outputDir = outputDir.replace("{folderName}", dir.toString()); + logger.info("outputDir 1={}", outputDir); + outputDir = outputDir.replace("\\watchedFolders", ""); + outputDir = outputDir.replace("//watchedFolders", ""); + outputDir = outputDir.replace("\\\\watchedFolders", ""); + outputDir = outputDir.replace("/watchedFolders", ""); + + Path outputPath; + logger.info("outputDir 2={}", outputDir); + if (Paths.get(outputDir).isAbsolute()) { + // If it's an absolute path, use it directly + outputPath = Paths.get(outputDir); + } else { + // If it's a relative path, make it relative to the current working directory + outputPath = Paths.get(".", outputDir); + } + + logger.info("outputPath={}", outputPath); + + if (!Files.exists(outputPath)) { + try { + Files.createDirectories(outputPath); + logger.info("Created directory: {}", outputPath); + } catch (IOException e) { + logger.error("Error creating directory: {}", outputPath, e); + return; + } + } + logger.info("outputPath {}", outputPath); + logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString()); + File newFile = new File(outputPath.resolve(outputFileName).toString()); + OutputStream os = new FileOutputStream(newFile); + os.write(((ByteArrayResource)resource).getByteArray()); + os.close(); + logger.info("made {}", outputPath.resolve(outputFileName)); + } + + // If successful, delete the original files + for (File file : filesToProcess) { + Files.deleteIfExists(processingDir.resolve(file.getName())); + } + } catch (Exception e) { + // If an error occurs, move the original files back + for (File file : filesToProcess) { + Files.move(processingDir.resolve(file.getName()), file.toPath()); + } + throw e; + } + } + } + } + + List processFiles(List outputFiles, String jsonString) throws Exception { + + logger.info("Processing files... " + outputFiles); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(jsonString); + + JsonNode pipelineNode = jsonNode.get("pipeline"); + ByteArrayOutputStream logStream = new ByteArrayOutputStream(); + PrintStream logPrintStream = new PrintStream(logStream); + + boolean hasErrors = false; + + for (JsonNode operationNode : pipelineNode) { + String operation = operationNode.get("operation").asText(); + logger.info("Running operation: {}", operation); + JsonNode parametersNode = operationNode.get("parameters"); + String inputFileExtension = ""; + if (operationNode.has("inputFileType")) { + inputFileExtension = operationNode.get("inputFileType").asText(); + } else { + inputFileExtension = ".pdf"; + } + + List newOutputFiles = new ArrayList<>(); + boolean hasInputFileType = false; + + for (Resource file : outputFiles) { + if (file.getFilename().endsWith(inputFileExtension)) { + hasInputFileType = true; + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("fileInput", file); + + Iterator> parameters = parametersNode.fields(); + while (parameters.hasNext()) { + Map.Entry parameter = parameters.next(); + body.add(parameter.getKey(), parameter.getValue().asText()); + } + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + HttpEntity> entity = new HttpEntity<>(body, headers); + + RestTemplate restTemplate = new RestTemplate(); + String url = "http://localhost:8080/" + operation; + + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class); + + if (!response.getStatusCode().equals(HttpStatus.OK)) { + logPrintStream.println("Error: " + response.getBody()); + hasErrors = true; + continue; + } + + // Check if the response body is a zip file + if (isZip(response.getBody())) { + // Unzip the file and add all the files to the new output files + newOutputFiles.addAll(unzip(response.getBody())); + } else { + Resource outputResource = new ByteArrayResource(response.getBody()) { + @Override + public String getFilename() { + return file.getFilename(); // Preserving original filename + } + }; + newOutputFiles.add(outputResource); + } + } + + if (!hasInputFileType) { + logPrintStream.println( + "No files with extension " + inputFileExtension + " found for operation " + operation); + hasErrors = true; + } + + outputFiles = newOutputFiles; + } + logPrintStream.close(); + + } + if (hasErrors) { + logger.error("Errors occurred during processing. Log: {}", logStream.toString()); + } + return outputFiles; + } + + List handleFiles(File[] files, String jsonString) throws Exception { + if(files == null || files.length == 0) { + logger.info("No files"); + return null; + } + + logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(jsonString); + + JsonNode pipelineNode = jsonNode.get("pipeline"); + + boolean hasErrors = false; + List outputFiles = new ArrayList<>(); + + for (File file : files) { + Path path = Paths.get(file.getAbsolutePath()); + System.out.println("Reading file: " + path); // debug statement + + if (Files.exists(path)) { + Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) { + @Override + public String getFilename() { + return file.getName(); + } + }; + outputFiles.add(fileResource); + } else { + System.out.println("File not found: " + path); // debug statement + } + } + logger.info("Files successfully loaded. Starting processing..."); + return processFiles(outputFiles, jsonString); + } + + List handleFiles(MultipartFile[] files, String jsonString) throws Exception { + if(files == null || files.length == 0) { + logger.info("No files"); + return null; + } + logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(jsonString); + + JsonNode pipelineNode = jsonNode.get("pipeline"); + + boolean hasErrors = false; + List outputFiles = new ArrayList<>(); + + for (MultipartFile file : files) { + Resource fileResource = new ByteArrayResource(file.getBytes()) { + @Override + public String getFilename() { + return file.getOriginalFilename(); + } + }; + outputFiles.add(fileResource); + } + logger.info("Files successfully loaded. Starting processing..."); + return processFiles(outputFiles, jsonString); + } + + @PostMapping("/handleData") + public ResponseEntity handleData(@RequestPart("fileInput") MultipartFile[] files, + @RequestParam("json") String jsonString) { + logger.info("Received POST request to /handleData with {} files", files.length); + try { + List outputFiles = handleFiles(files, jsonString); + + if (outputFiles != null && outputFiles.size() == 1) { + // If there is only one file, return it directly + Resource singleFile = outputFiles.get(0); + InputStream is = singleFile.getInputStream(); + byte[] bytes = new byte[(int) singleFile.contentLength()]; + is.read(bytes); + is.close(); + + logger.info("Returning single file response..."); + return WebResponseUtils.bytesToWebResponse(bytes, singleFile.getFilename(), + MediaType.APPLICATION_OCTET_STREAM); + } else if (outputFiles == null) { + return null; + } + + // Create a ByteArrayOutputStream to hold the zip + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ZipOutputStream zipOut = new ZipOutputStream(baos); + + // Loop through each file and add it to the zip + for (Resource file : outputFiles) { + ZipEntry zipEntry = new ZipEntry(file.getFilename()); + zipOut.putNextEntry(zipEntry); + + // Read the file into a byte array + InputStream is = file.getInputStream(); + byte[] bytes = new byte[(int) file.contentLength()]; + is.read(bytes); + + // Write the bytes of the file to the zip + zipOut.write(bytes, 0, bytes.length); + zipOut.closeEntry(); + + is.close(); + } + + zipOut.close(); + + logger.info("Returning zipped file response..."); + return WebResponseUtils.boasToWebResponse(baos, "output.zip", MediaType.APPLICATION_OCTET_STREAM); + } catch (Exception e) { + logger.error("Error handling data: ", e); + return null; + } + } + + private boolean isZip(byte[] data) { + if (data == null || data.length < 4) { + return false; + } + + // Check the first four bytes of the data against the standard zip magic number + return data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04; + } + + private List unzip(byte[] data) throws IOException { + logger.info("Unzipping data of length: {}", data.length); + List unzippedFiles = new ArrayList<>(); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(data); + ZipInputStream zis = new ZipInputStream(bais)) { + + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int count; + + while ((count = zis.read(buffer)) != -1) { + baos.write(buffer, 0, count); + } + + final String filename = entry.getName(); + Resource fileResource = new ByteArrayResource(baos.toByteArray()) { + @Override + public String getFilename() { + return filename; + } + }; + + // If the unzipped file is a zip file, unzip it + if (isZip(baos.toByteArray())) { + logger.info("File {} is a zip file. Unzipping...", filename); + unzippedFiles.addAll(unzip(baos.toByteArray())); + } else { + unzippedFiles.add(fileResource); + } + } + } + + logger.info("Unzipping completed. {} files were unzipped.", unzippedFiles.size()); + return unzippedFiles; + } + +} diff --git a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java index 4548e9c3..31687299 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java @@ -1,28 +1,76 @@ package stirling.software.SPDF.controller.web; +import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.GetMapping; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.HashMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.ModelAttribute; + +import com.fasterxml.jackson.databind.ObjectMapper; + import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.tags.Tag; - @Controller @Tag(name = "General", description = "General APIs") public class GeneralWebController { - @GetMapping("/pipeline") - @Hidden - public String pipelineForm(Model model) { - model.addAttribute("currentPage", "pipeline"); - return "pipeline"; + + @GetMapping("/pipeline") + @Hidden + public String pipelineForm(Model model) { + model.addAttribute("currentPage", "pipeline"); + + List pipelineConfigs = new ArrayList<>(); + try (Stream paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) { + List jsonFiles = paths + .filter(Files::isRegularFile) + .filter(p -> p.toString().endsWith(".json")) + .collect(Collectors.toList()); + + for (Path jsonFile : jsonFiles) { + String content = Files.readString(jsonFile, StandardCharsets.UTF_8); + pipelineConfigs.add(content); + } + List> pipelineConfigsWithNames = new ArrayList<>(); + for (String config : pipelineConfigs) { + Map jsonContent = new ObjectMapper().readValue(config, Map.class); + String name = (String) jsonContent.get("name"); + Map configWithName = new HashMap<>(); + configWithName.put("json", config); + configWithName.put("name", name); + pipelineConfigsWithNames.add(configWithName); + } + model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames); + + } catch (IOException e) { + e.printStackTrace(); } + + model.addAttribute("pipelineConfigs", pipelineConfigs); + + return "pipeline"; + } + + + @GetMapping("/merge-pdfs") @Hidden diff --git a/src/main/resources/static/js/pipeline.js b/src/main/resources/static/js/pipeline.js index ad77d970..85918cd5 100644 --- a/src/main/resources/static/js/pipeline.js +++ b/src/main/resources/static/js/pipeline.js @@ -1,432 +1,456 @@ -document.getElementById('validateButton').addEventListener('click', function(event) { - event.preventDefault(); - validatePipeline(); -}); -function validatePipeline() { - let pipelineListItems = document.getElementById('pipelineList').children; - let isValid = true; - let containsAddPassword = false; - for (let i = 0; i < pipelineListItems.length - 1; i++) { - let currentOperation = pipelineListItems[i].querySelector('.operationName').textContent; - let nextOperation = pipelineListItems[i + 1].querySelector('.operationName').textContent; - if (currentOperation === '/add-password') { - containsAddPassword = true; - } - console.log(currentOperation); - console.log(apiDocs[currentOperation]); - let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; - let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; - - console.log("currentOperationDescription", currentOperationDescription); - console.log("nextOperationDescription", nextOperationDescription); - - let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; - let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; - - console.log("Operation " + currentOperation + " Output: " + currentOperationOutput); - console.log("Operation " + nextOperation + " Input: " + nextOperationInput); - - // Splitting in case of multiple possible output/input - let currentOperationOutputArr = currentOperationOutput.split('/'); - let nextOperationInputArr = nextOperationInput.split('/'); - - if (currentOperationOutput !== 'ANY' && nextOperationInput !== 'ANY') { - let intersection = currentOperationOutputArr.filter(value => nextOperationInputArr.includes(value)); - console.log(`Intersection: ${intersection}`); - - if (intersection.length === 0) { - isValid = false; - console.log(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); - alert(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); - break; - } - } - } - if (containsAddPassword && pipelineListItems[pipelineListItems.length - 1].querySelector('.operationName').textContent !== '/add-password') { - alert('The "add-password" operation should be at the end of the operations sequence. Please adjust the operations order.'); - return false; - } - if (isValid) { - console.log('Pipeline is valid'); - // Continue with the pipeline operation - } else { - console.error('Pipeline is not valid'); - // Stop operation, maybe display an error to the user - } - - return isValid; -} - - - - - -document.getElementById('submitConfigBtn').addEventListener('click', function() { - - if (validatePipeline() === false) { - return; - } - let selectedOperation = document.getElementById('operationsDropdown').value; - let parameters = operationSettings[selectedOperation] || {}; - - let pipelineConfig = { - "name": "uniquePipelineName", - "pipeline": [{ - "operation": selectedOperation, - "parameters": parameters - }], - "_examples": { - "outputDir" : "{outputFolder}/{folderName}", - "outputFileName" : "{filename}-{pipelineName}-{date}-{time}" - }, - "outputDir" : "httpWebRequest", - "outputFileName" : "{filename}" - }; - - let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2); - - let formData = new FormData(); - - let fileInput = document.getElementById('fileInput'); - let files = fileInput.files; - - for (let i = 0; i < files.length; i++) { - console.log("files[i]", files[i].name); - formData.append('fileInput', files[i], files[i].name); - } - - console.log("pipelineConfigJson", pipelineConfigJson); - formData.append('json', pipelineConfigJson); - console.log("formData", formData); - - fetch('/handleData', { - method: 'POST', - body: formData - }) - .then(response => response.blob()) - .then(blob => { - - let url = window.URL.createObjectURL(blob); - let a = document.createElement('a'); - a.href = url; - a.download = 'outputfile'; - document.body.appendChild(a); - a.click(); - a.remove(); - }) - .catch((error) => { - console.error('Error:', error); - }); -}); - -let apiDocs = {}; - -let operationSettings = {}; - -fetch('v3/api-docs') - .then(response => response.json()) - .then(data => { - - apiDocs = data.paths; - let operationsDropdown = document.getElementById('operationsDropdown'); - const ignoreOperations = ["/handleData", "operationToIgnore"]; // Add the operations you want to ignore here - - operationsDropdown.innerHTML = ''; - - let operationsByTag = {}; - - // Group operations by tags - Object.keys(data.paths).forEach(operationPath => { - let operation = data.paths[operationPath].post; - if (operation && !ignoreOperations.includes(operationPath) && !operation.description.includes("Type:MISO")) { - let operationTag = operation.tags[0]; // This assumes each operation has exactly one tag - if (!operationsByTag[operationTag]) { - operationsByTag[operationTag] = []; - } - operationsByTag[operationTag].push(operationPath); - } - }); - - // Specify the order of tags - let tagOrder = ["General", "Security", "Convert", "Other", "Filter"]; - - // Create dropdown options - tagOrder.forEach(tag => { - if (operationsByTag[tag]) { - let group = document.createElement('optgroup'); - group.label = tag; - - operationsByTag[tag].forEach(operationPath => { - let option = document.createElement('option'); - let operationWithoutSlash = operationPath.replace(/\//g, ''); // Remove slashes - option.textContent = operationWithoutSlash; - option.value = operationPath; // Keep the value with slashes for querying - group.appendChild(option); - }); - - operationsDropdown.appendChild(group); - } - }); - }); - - -document.getElementById('addOperationBtn').addEventListener('click', function() { - let selectedOperation = document.getElementById('operationsDropdown').value; - let pipelineList = document.getElementById('pipelineList'); - - let listItem = document.createElement('li'); - listItem.className = "list-group-item"; - let hasSettings = (apiDocs[selectedOperation] && apiDocs[selectedOperation].post && - apiDocs[selectedOperation].post.parameters && apiDocs[selectedOperation].post.parameters.length > 0); - - - listItem.innerHTML = ` -
-
${selectedOperation}
-
- - - - -
-
- `; - - pipelineList.appendChild(listItem); - - listItem.querySelector('.move-up').addEventListener('click', function(event) { - event.preventDefault(); - if (listItem.previousElementSibling) { - pipelineList.insertBefore(listItem, listItem.previousElementSibling); - } - }); - - listItem.querySelector('.move-down').addEventListener('click', function(event) { - event.preventDefault(); - if (listItem.nextElementSibling) { - pipelineList.insertBefore(listItem.nextElementSibling, listItem); - } - }); - - listItem.querySelector('.remove').addEventListener('click', function(event) { - event.preventDefault(); - pipelineList.removeChild(listItem); - }); - - listItem.querySelector('.pipelineSettings').addEventListener('click', function(event) { - event.preventDefault(); - showpipelineSettingsModal(selectedOperation); - }); - - function showpipelineSettingsModal(operation) { - let pipelineSettingsModal = document.getElementById('pipelineSettingsModal'); - let pipelineSettingsContent = document.getElementById('pipelineSettingsContent'); - let operationData = apiDocs[operation].post.parameters || []; - - pipelineSettingsContent.innerHTML = ''; - - operationData.forEach(parameter => { - let parameterDiv = document.createElement('div'); - parameterDiv.className = "form-group"; - - let parameterLabel = document.createElement('label'); - parameterLabel.textContent = `${parameter.name} (${parameter.schema.type}): `; - parameterLabel.title = parameter.description; - parameterDiv.appendChild(parameterLabel); - - let parameterInput; - switch (parameter.schema.type) { - case 'string': - case 'number': - case 'integer': - parameterInput = document.createElement('input'); - parameterInput.type = parameter.schema.type === 'string' ? 'text' : 'number'; - parameterInput.className = "form-control"; - break; - case 'boolean': - parameterInput = document.createElement('input'); - parameterInput.type = 'checkbox'; - break; - case 'array': - case 'object': - parameterInput = document.createElement('textarea'); - parameterInput.placeholder = `Enter a JSON formatted ${parameter.schema.type}`; - parameterInput.className = "form-control"; - break; - case 'enum': - parameterInput = document.createElement('select'); - parameterInput.className = "form-control"; - parameter.schema.enum.forEach(option => { - let optionElement = document.createElement('option'); - optionElement.value = option; - optionElement.text = option; - parameterInput.appendChild(optionElement); - }); - break; - default: - parameterInput = document.createElement('input'); - parameterInput.type = 'text'; - parameterInput.className = "form-control"; - } - parameterInput.id = parameter.name; - - if (operationSettings[operation] && operationSettings[operation][parameter.name] !== undefined) { - let savedValue = operationSettings[operation][parameter.name]; - - switch (parameter.schema.type) { - case 'number': - case 'integer': - parameterInput.value = savedValue.toString(); - break; - case 'boolean': - parameterInput.checked = savedValue; - break; - case 'array': - case 'object': - parameterInput.value = JSON.stringify(savedValue); - break; - default: - parameterInput.value = savedValue; - } - } - - parameterDiv.appendChild(parameterInput); - - pipelineSettingsContent.appendChild(parameterDiv); - }); - - let saveButton = document.createElement('button'); - saveButton.textContent = "Save Settings"; - saveButton.className = "btn btn-primary"; - saveButton.addEventListener('click', function(event) { - event.preventDefault(); - let settings = {}; - operationData.forEach(parameter => { - let value = document.getElementById(parameter.name).value; - switch (parameter.schema.type) { - case 'number': - case 'integer': - settings[parameter.name] = Number(value); - break; - case 'boolean': - settings[parameter.name] = document.getElementById(parameter.name).checked; - break; - case 'array': - case 'object': - try { - settings[parameter.name] = JSON.parse(value); - } catch (err) { - console.error(`Invalid JSON format for ${parameter.name}`); - } - break; - default: - settings[parameter.name] = value; - } - }); - operationSettings[operation] = settings; - console.log(settings); - pipelineSettingsModal.style.display = "none"; - }); - pipelineSettingsContent.appendChild(saveButton); - - pipelineSettingsModal.style.display = "block"; - - pipelineSettingsModal.getElementsByClassName("close")[0].onclick = function() { - pipelineSettingsModal.style.display = "none"; - } - - window.onclick = function(event) { - if (event.target == pipelineSettingsModal) { - pipelineSettingsModal.style.display = "none"; - } - } - } - - document.getElementById('savePipelineBtn').addEventListener('click', function() { - if (validatePipeline() === false) { - return; - } - var pipelineName = document.getElementById('pipelineName').value; - let pipelineList = document.getElementById('pipelineList').children; - let pipelineConfig = { - "name": pipelineName, - "pipeline": [], - "_examples": { - "outputDir" : "{outputFolder}/{folderName}", - "outputFileName" : "{filename}-{pipelineName}-{date}-{time}" - }, - "outputDir" : "httpWebRequest", - "outputFileName" : "{filename}" - }; - - for (let i = 0; i < pipelineList.length; i++) { - let operationName = pipelineList[i].querySelector('.operationName').textContent; - let parameters = operationSettings[operationName] || {}; - - pipelineConfig.pipeline.push({ - "operation": operationName, - "parameters": parameters - }); - } - - let a = document.createElement('a'); - a.href = URL.createObjectURL(new Blob([JSON.stringify(pipelineConfig, null, 2)], { - type: 'application/json' - })); - a.download = 'pipelineConfig.json'; - a.style.display = 'none'; - - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - }); - - document.getElementById('uploadPipelineBtn').addEventListener('click', function() { - document.getElementById('uploadPipelineInput').click(); - }); - - document.getElementById('uploadPipelineInput').addEventListener('change', function(e) { - let reader = new FileReader(); - reader.onload = function(event) { - let pipelineConfig = JSON.parse(event.target.result); - let pipelineList = document.getElementById('pipelineList'); - - while (pipelineList.firstChild) { - pipelineList.removeChild(pipelineList.firstChild); - } - document.getElementById('pipelineName').value = pipelineConfig.name - pipelineConfig.pipeline.forEach(operationConfig => { - let operationsDropdown = document.getElementById('operationsDropdown'); - operationsDropdown.value = operationConfig.operation; - operationSettings[operationConfig.operation] = operationConfig.parameters; - document.getElementById('addOperationBtn').click(); - - let lastOperation = pipelineList.lastChild; - - lastOperation.querySelector('.pipelineSettings').click(); - - Object.keys(operationConfig.parameters).forEach(parameterName => { - let input = document.getElementById(parameterName); - if (input) { - switch (input.type) { - case 'checkbox': - input.checked = operationConfig.parameters[parameterName]; - break; - case 'number': - input.value = operationConfig.parameters[parameterName].toString(); - break; - case 'text': - case 'textarea': - default: - input.value = JSON.stringify(operationConfig.parameters[parameterName]); - } - } - }); - - document.querySelector('#pipelineSettingsModal .btn-primary').click(); - }); - }; - reader.readAsText(e.target.files[0]); - }); - +document.getElementById('validateButton').addEventListener('click', function(event) { + event.preventDefault(); + validatePipeline(); +}); +function validatePipeline() { + let pipelineListItems = document.getElementById('pipelineList').children; + let isValid = true; + let containsAddPassword = false; + for (let i = 0; i < pipelineListItems.length - 1; i++) { + let currentOperation = pipelineListItems[i].querySelector('.operationName').textContent; + let nextOperation = pipelineListItems[i + 1].querySelector('.operationName').textContent; + if (currentOperation === '/add-password') { + containsAddPassword = true; + } + console.log(currentOperation); + console.log(apiDocs[currentOperation]); + let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; + let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; + + console.log("currentOperationDescription", currentOperationDescription); + console.log("nextOperationDescription", nextOperationDescription); + + let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; + let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; + + console.log("Operation " + currentOperation + " Output: " + currentOperationOutput); + console.log("Operation " + nextOperation + " Input: " + nextOperationInput); + + // Splitting in case of multiple possible output/input + let currentOperationOutputArr = currentOperationOutput.split('/'); + let nextOperationInputArr = nextOperationInput.split('/'); + + if (currentOperationOutput !== 'ANY' && nextOperationInput !== 'ANY') { + let intersection = currentOperationOutputArr.filter(value => nextOperationInputArr.includes(value)); + console.log(`Intersection: ${intersection}`); + + if (intersection.length === 0) { + isValid = false; + console.log(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); + alert(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); + break; + } + } + } + if (containsAddPassword && pipelineListItems[pipelineListItems.length - 1].querySelector('.operationName').textContent !== '/add-password') { + alert('The "add-password" operation should be at the end of the operations sequence. Please adjust the operations order.'); + return false; + } + if (isValid) { + console.log('Pipeline is valid'); + // Continue with the pipeline operation + } else { + console.error('Pipeline is not valid'); + // Stop operation, maybe display an error to the user + } + + return isValid; +} + + + + + +document.getElementById('submitConfigBtn').addEventListener('click', function() { + + if (validatePipeline() === false) { + return; + } + let selectedOperation = document.getElementById('operationsDropdown').value; + let parameters = operationSettings[selectedOperation] || {}; + + let pipelineConfig = { + "name": "uniquePipelineName", + "pipeline": [{ + "operation": selectedOperation, + "parameters": parameters + }], + "_examples": { + "outputDir" : "{outputFolder}/{folderName}", + "outputFileName" : "{filename}-{pipelineName}-{date}-{time}" + }, + "outputDir" : "httpWebRequest", + "outputFileName" : "{filename}" + }; + + let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2); + + let formData = new FormData(); + + let fileInput = document.getElementById('fileInput'); + let files = fileInput.files; + + for (let i = 0; i < files.length; i++) { + console.log("files[i]", files[i].name); + formData.append('fileInput', files[i], files[i].name); + } + + console.log("pipelineConfigJson", pipelineConfigJson); + formData.append('json', pipelineConfigJson); + console.log("formData", formData); + + fetch('/handleData', { + method: 'POST', + body: formData + }) + .then(response => response.blob()) + .then(blob => { + + let url = window.URL.createObjectURL(blob); + let a = document.createElement('a'); + a.href = url; + a.download = 'outputfile'; + document.body.appendChild(a); + a.click(); + a.remove(); + }) + .catch((error) => { + console.error('Error:', error); + }); +}); + +let apiDocs = {}; + +let operationSettings = {}; + +fetch('v3/api-docs') + .then(response => response.json()) + .then(data => { + + apiDocs = data.paths; + let operationsDropdown = document.getElementById('operationsDropdown'); + const ignoreOperations = ["/handleData", "operationToIgnore"]; // Add the operations you want to ignore here + + operationsDropdown.innerHTML = ''; + + let operationsByTag = {}; + + // Group operations by tags + Object.keys(data.paths).forEach(operationPath => { + let operation = data.paths[operationPath].post; + if (operation && !ignoreOperations.includes(operationPath) && !operation.description.includes("Type:MISO")) { + let operationTag = operation.tags[0]; // This assumes each operation has exactly one tag + if (!operationsByTag[operationTag]) { + operationsByTag[operationTag] = []; + } + operationsByTag[operationTag].push(operationPath); + } + }); + + // Specify the order of tags + let tagOrder = ["General", "Security", "Convert", "Other", "Filter"]; + + // Create dropdown options + tagOrder.forEach(tag => { + if (operationsByTag[tag]) { + let group = document.createElement('optgroup'); + group.label = tag; + + operationsByTag[tag].forEach(operationPath => { + let option = document.createElement('option'); + let operationWithoutSlash = operationPath.replace(/\//g, ''); // Remove slashes + option.textContent = operationWithoutSlash; + option.value = operationPath; // Keep the value with slashes for querying + group.appendChild(option); + }); + + operationsDropdown.appendChild(group); + } + }); + }); + + +document.getElementById('addOperationBtn').addEventListener('click', function() { + let selectedOperation = document.getElementById('operationsDropdown').value; + let pipelineList = document.getElementById('pipelineList'); + + let listItem = document.createElement('li'); + listItem.className = "list-group-item"; + let hasSettings = (apiDocs[selectedOperation] && apiDocs[selectedOperation].post && + apiDocs[selectedOperation].post.parameters && apiDocs[selectedOperation].post.parameters.length > 0); + + + listItem.innerHTML = ` +
+
${selectedOperation}
+
+ + + + +
+
+ `; + + pipelineList.appendChild(listItem); + + listItem.querySelector('.move-up').addEventListener('click', function(event) { + event.preventDefault(); + if (listItem.previousElementSibling) { + pipelineList.insertBefore(listItem, listItem.previousElementSibling); + } + }); + + listItem.querySelector('.move-down').addEventListener('click', function(event) { + event.preventDefault(); + if (listItem.nextElementSibling) { + pipelineList.insertBefore(listItem.nextElementSibling, listItem); + } + }); + + listItem.querySelector('.remove').addEventListener('click', function(event) { + event.preventDefault(); + pipelineList.removeChild(listItem); + }); + + listItem.querySelector('.pipelineSettings').addEventListener('click', function(event) { + event.preventDefault(); + showpipelineSettingsModal(selectedOperation); + }); + + function showpipelineSettingsModal(operation) { + let pipelineSettingsModal = document.getElementById('pipelineSettingsModal'); + let pipelineSettingsContent = document.getElementById('pipelineSettingsContent'); + let operationData = apiDocs[operation].post.parameters || []; + + pipelineSettingsContent.innerHTML = ''; + + operationData.forEach(parameter => { + let parameterDiv = document.createElement('div'); + parameterDiv.className = "form-group"; + + let parameterLabel = document.createElement('label'); + parameterLabel.textContent = `${parameter.name} (${parameter.schema.type}): `; + parameterLabel.title = parameter.description; + parameterDiv.appendChild(parameterLabel); + + let parameterInput; + switch (parameter.schema.type) { + case 'string': + case 'number': + case 'integer': + parameterInput = document.createElement('input'); + parameterInput.type = parameter.schema.type === 'string' ? 'text' : 'number'; + parameterInput.className = "form-control"; + break; + case 'boolean': + parameterInput = document.createElement('input'); + parameterInput.type = 'checkbox'; + break; + case 'array': + case 'object': + parameterInput = document.createElement('textarea'); + parameterInput.placeholder = `Enter a JSON formatted ${parameter.schema.type}`; + parameterInput.className = "form-control"; + break; + case 'enum': + parameterInput = document.createElement('select'); + parameterInput.className = "form-control"; + parameter.schema.enum.forEach(option => { + let optionElement = document.createElement('option'); + optionElement.value = option; + optionElement.text = option; + parameterInput.appendChild(optionElement); + }); + break; + default: + parameterInput = document.createElement('input'); + parameterInput.type = 'text'; + parameterInput.className = "form-control"; + } + parameterInput.id = parameter.name; + + if (operationSettings[operation] && operationSettings[operation][parameter.name] !== undefined) { + let savedValue = operationSettings[operation][parameter.name]; + + switch (parameter.schema.type) { + case 'number': + case 'integer': + parameterInput.value = savedValue.toString(); + break; + case 'boolean': + parameterInput.checked = savedValue; + break; + case 'array': + case 'object': + parameterInput.value = JSON.stringify(savedValue); + break; + default: + parameterInput.value = savedValue; + } + } + + parameterDiv.appendChild(parameterInput); + + pipelineSettingsContent.appendChild(parameterDiv); + }); + + let saveButton = document.createElement('button'); + saveButton.textContent = "Save Settings"; + saveButton.className = "btn btn-primary"; + saveButton.addEventListener('click', function(event) { + event.preventDefault(); + let settings = {}; + operationData.forEach(parameter => { + let value = document.getElementById(parameter.name).value; + switch (parameter.schema.type) { + case 'number': + case 'integer': + settings[parameter.name] = Number(value); + break; + case 'boolean': + settings[parameter.name] = document.getElementById(parameter.name).checked; + break; + case 'array': + case 'object': + try { + settings[parameter.name] = JSON.parse(value); + } catch (err) { + console.error(`Invalid JSON format for ${parameter.name}`); + } + break; + default: + settings[parameter.name] = value; + } + }); + operationSettings[operation] = settings; + console.log(settings); + pipelineSettingsModal.style.display = "none"; + }); + pipelineSettingsContent.appendChild(saveButton); + + pipelineSettingsModal.style.display = "block"; + + pipelineSettingsModal.getElementsByClassName("close")[0].onclick = function() { + pipelineSettingsModal.style.display = "none"; + } + + window.onclick = function(event) { + if (event.target == pipelineSettingsModal) { + pipelineSettingsModal.style.display = "none"; + } + } + } + + document.getElementById('savePipelineBtn').addEventListener('click', function() { + if (validatePipeline() === false) { + return; + } + var pipelineName = document.getElementById('pipelineName').value; + let pipelineList = document.getElementById('pipelineList').children; + let pipelineConfig = { + "name": pipelineName, + "pipeline": [], + "_examples": { + "outputDir" : "{outputFolder}/{folderName}", + "outputFileName" : "{filename}-{pipelineName}-{date}-{time}" + }, + "outputDir" : "httpWebRequest", + "outputFileName" : "{filename}" + }; + + for (let i = 0; i < pipelineList.length; i++) { + let operationName = pipelineList[i].querySelector('.operationName').textContent; + let parameters = operationSettings[operationName] || {}; + + pipelineConfig.pipeline.push({ + "operation": operationName, + "parameters": parameters + }); + } + + let a = document.createElement('a'); + a.href = URL.createObjectURL(new Blob([JSON.stringify(pipelineConfig, null, 2)], { + type: 'application/json' + })); + a.download = 'pipelineConfig.json'; + a.style.display = 'none'; + + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }); + + async function processPipelineConfig(configString) { + let pipelineConfig = JSON.parse(configString); + let pipelineList = document.getElementById('pipelineList'); + + while (pipelineList.firstChild) { + pipelineList.removeChild(pipelineList.firstChild); + } + document.getElementById('pipelineName').value = pipelineConfig.name + for (const operationConfig of pipelineConfig.pipeline) { + let operationsDropdown = document.getElementById('operationsDropdown'); + operationsDropdown.value = operationConfig.operation; + operationSettings[operationConfig.operation] = operationConfig.parameters; + + // assuming addOperation is async + await new Promise((resolve) => { + document.getElementById('addOperationBtn').addEventListener('click', resolve, { once: true }); + document.getElementById('addOperationBtn').click(); + }); + + let lastOperation = pipelineList.lastChild; + + Object.keys(operationConfig.parameters).forEach(parameterName => { + let input = document.getElementById(parameterName); + if (input) { + switch (input.type) { + case 'checkbox': + input.checked = operationConfig.parameters[parameterName]; + break; + case 'number': + input.value = operationConfig.parameters[parameterName].toString(); + break; + case 'file': + if (parameterName !== 'fileInput') { + // Create a new file input element + let newInput = document.createElement('input'); + newInput.type = 'file'; + newInput.id = parameterName; + + // Add the new file input to the main page (change the selector according to your needs) + document.querySelector('#main').appendChild(newInput); + } + break; + case 'text': + case 'textarea': + default: + input.value = JSON.stringify(operationConfig.parameters[parameterName]); + } + } + }); + + } +} + + + document.getElementById('uploadPipelineBtn').addEventListener('click', function() { + document.getElementById('uploadPipelineInput').click(); + }); + + document.getElementById('uploadPipelineInput').addEventListener('change', function(e) { + let reader = new FileReader(); + reader.onload = function(event) { + processPipelineConfig(event.target.result); + }; + reader.readAsText(e.target.files[0]); + }); + + document.getElementById('pipelineSelect').addEventListener('change', function(e) { + let selectedPipelineJson = e.target.value; // assuming the selected value is the JSON string of the pipeline config + processPipelineConfig(selectedPipelineJson); + }); + + }); \ No newline at end of file diff --git a/src/main/resources/templates/pipeline.html b/src/main/resources/templates/pipeline.html index c1ea9354..dc959a39 100644 --- a/src/main/resources/templates/pipeline.html +++ b/src/main/resources/templates/pipeline.html @@ -1,115 +1,102 @@ - - - - - -
-
-
-

-
-
-
- -
- - - -
- - -
-
- -
- - -
-

Pipeline Configuration

-
- - -
-
- - -
-
- -
-
- -
-

Pipeline:

-
    - -
-
- - - - - - -
- - - -
-
-
-
- -
-
- - + + + + + +
+
+
+

+
+
+ + + + + + + + + + + + + + + + + + + + + +
+
+
+ +
+
+ + \ No newline at end of file