Java Rsync Escape Spaces

Question

I am trying to run rsync from a jar. When the source path has no spaces it all works fine, but when the source path has a space in it, it fails. I have tried various methods for escaping the spaces, as per the man pages, such as source.replaceAll("\s", "\\ ") or source.replaceAll("\s", "?"), but to no avail.

When I output the command that is run and then run the exact same command from the command line, it all works. I can't see what I am doing wrong

My code is as follows:

RsyncCommandLine class

public class RsyncCommandLine {

    /** Logger */
    private static final Logger logger = LogManager.getLogger(RsyncCommandLine.class);

    private CommandLine commandLine = null;

    public String startRsync(String source, String destination) throws IOException {
        commandLine = createCommandLine(source, destination);

        CommandLineExecutorHelper helper = new CommandLineExecutorHelper();
        CommandLineLogOutputStream outputStream = helper.executeCommandLine(commandLine);
        validateResponse(outputStream);
        return convertLinesToString(outputStream.getLines());
    }

    private void validateResponse(CommandLineLogOutputStream outputStream) throws IOException {
        if (outputStream == null) {
            logger.error("outputStream  is not valid");
            throw new IOException("Unable to use rsync. ");
        } else if (outputStream.getExitCode() != 0) {
            logger.error("Exit code: " + outputStream.getExitCode());
            logger.error("Validate Response failed " + outputStream.getLines());
            String errorMessage = exitCodeToErrorMessage(outputStream.getExitCode());
            throw new IOException("Error with request. " + errorMessage);
        } else if (outputStream.getLines() == null || outputStream.getLines().isEmpty()) {
            logger.error("Rsync result: " + outputStream.getLines());
            String errorMessage = "Unable to rsync. ";

            throw new IOException(errorMessage);

        }
    }

    private String exitCodeToErrorMessage(int exitCode) {
        String errorMessage = null;
        switch (exitCode) {
            case 0: errorMessage="Success."; break;
            case 1: errorMessage="Syntax or usage error."; break;
            case 2: errorMessage="Protocol incompatibility."; break;
            case 3: errorMessage="Errors selecting input/output files, dirs."; break;
            case 4: errorMessage="Requested action not supported: an attempt was made to manipulate 64-bit files on a platform that cannot support them; or an option was specified that is supported by the client and not by the server."; break;
            case 5: errorMessage="Error starting client-server protocol."; break;
            case 6: errorMessage="Daemon unable to append to log-file."; break;
            case 10: errorMessage="Error in socket I/O."; break;
            case 11: errorMessage="Error in file I/O."; break;
            case 12: errorMessage="Error in rsync protocol data stream."; break;
            case 13: errorMessage="Errors with program diagnostics."; break;
            case 14: errorMessage="Error in IPC code."; break;
            case 20: errorMessage="Received SIGUSR1 or SIGINT."; break;
            case 21: errorMessage="Some error returned by waitpid()."; break;
            case 22: errorMessage="Error allocating core memory buffers."; break;
            case 23: errorMessage="Partial transfer due to error."; break;
            case 24: errorMessage="Partial transfer due to vanished source files."; break;
            case 25: errorMessage="The --max-delete limit stopped deletions."; break;
            case 30: errorMessage="Timeout in data send/receive."; break;
            case 35: errorMessage="Timeout waiting for daemon connection."; break;
            default: errorMessage="Unrecognised error code.";
        }
        return errorMessage;
    }


    protected String convertLinesToString(List<String> lines) {
        String result = null;

        if (lines != null && !lines.isEmpty()) {
            StringBuilder builder = new StringBuilder();
            for (String line : lines) {
                builder.append(line).append(" ");
            }
            result = builder.toString().trim();
        }
        return result;
    }

    protected CommandLine createCommandLine(String source, String destination) {
        // rsync -rtvuch <source> <destination>

        commandLine = new CommandLine("rsync");
        commandLine.addArgument("-rtvuch");

        String escapedSource = source.trim().replaceAll("\\s", "\\\\ ");
        String escapedDestination = destination.trim().replaceAll("\\s", "\\\\ ");
        commandLine.addArgument(source);
        commandLine.addArgument(escapedDestination);

        logger.debug("escapedSource " + escapedSource);
        logger.debug("escapedDestination " + escapedDestination);

        return commandLine;
    }

}

CommandLineExecutorHelper class -

public class CommandLineExecutorHelper {

    /** Logger */
    private static final Logger logger = LogManager.getLogger(CommandLineExecutorHelper.class);

    private DefaultExecutor executor = new DefaultExecutor();

    private ExecuteWatchdog watchdog = new ExecuteWatchdog(10000);

    private DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler();


    public CommandLineExecutorHelper() {
        executor.setWatchdog(watchdog);
    }

    public CommandLineLogOutputStream executeCommandLine(CommandLine commandLine) {
        CommandLineLogOutputStream outputStream = new CommandLineLogOutputStream();
        PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(outputStream);
        executor.setStreamHandler(pumpStreamHandler);
        try {
            executor.execute(commandLine, resultHandler);

            resultHandler.waitFor();
            outputStream.setExitCode(resultHandler.getExitValue());
            logger.debug("\n\ncommandLine " + commandLine);
            logger.debug("exit code " + resultHandler.getExitValue());
            logger.debug("output " + outputStream.getLines());
        } catch (InterruptedException e) {
            outputStream.addErrorMessage(e.getMessage());
            logger.error("executeCommandLine " + e.getMessage());
        } catch (ExecuteException e) {
            outputStream.addErrorMessage(e.getMessage());
            logger.error("executeCommandLine " + e.getMessage());
        } catch (IOException e) {
            outputStream.addErrorMessage(e.getMessage());
            logger.error("executeCommandLine " + e.getMessage());
        } finally {
            IOUtils.closeQuietly(outputStream);
        }

        return outputStream;
    }
}

CommnadLineOutputStream class -

public class CommandLineLogOutputStream extends LogOutputStream {
    private int exitCode = -1;

    private final List<String> lines = new LinkedList<>();

    private StringBuilder errorMessages = new StringBuilder();


    /**
     * @return the exitCode
     */
    public int getExitCode() {
        return exitCode;
    }

    /**
     * @param exitCode the exitCode to set
     */
    public void setExitCode(int exitCode) {
        this.exitCode = exitCode;
    }

    /**
     * @return the lines
     */
    public List<String> getLines() {
        return lines;
    }



    /**
     * @return the errorMessages
     */
    public StringBuilder getErrorMessages() {
        return errorMessages;
    }

    /**
     * @param errorMessages the errorMessages to set
     */
    public void setErrorMessages(StringBuilder errorMessages) {
        this.errorMessages = errorMessages;
    }

    public void addErrorMessage(String errorMessage) {
        this.errorMessages.append(errorMessage);
    }


    @Override
    protected void processLine(String line, int logLevel) {
        lines.add(line);
    }

    /* (non-Javadoc)
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {
        StringBuilder builder = new StringBuilder();
        builder.append("CommandLineLogOutputStream [exitCode=").append(exitCode).append(", lines=").append(lines).append(", errorMessages=").append(errorMessages).append("]");
        return builder.toString();
    }

}

So when I run my jar without a space it is successful:

java -jar myjar.jar -source "/var/source"

output command is:

commandLine [rsync, -rtvuch, "/var/source", /var/dest]

When I run the same jar against a path with spaces:

java -jar myjar.jar -source "/var/source with spaces"

I get the following error message:

Exit code: 23
Validate Response failed [building file list ... donersync: link_stat "/Users/karen/"/var/source with spaces"" failed: No such file or directory (2), building file list ... donersync: link_stat "/Users/karen/"/var/source with spaces"" failed: No such file or directory (2), , sent 21 bytes  received 20 bytes  82.00 bytes/sec, total size is 0  speedup is 0.00, rsync error: some files could not be transferred (code 23) at /BuildRoot/Library/Caches/com.apple.xbs/Sources/rsync/rsync-47/rsync/main.c(992) [sender=2.6.9]]
Unable to rsync Error with request. Partial transfer due to error.

The destination path is picked up from a file open dialog.

Answer 1

After various input I decided to use the ProcessBuilder instead. I then got it working using the following code:

public class RsyncCommandLine {

    /** Logger */
    private static final Logger logger = LogManager.getLogger(RsyncCommandLine.class);


    public String startRsync(String source, String destination) throws IOException {
        List<String> commands = createCommandLine(source, destination);
        List<String> lines = new ArrayList<>();
        Integer exitCode = null;
        try {
            ProcessBuilder processBuilder = new ProcessBuilder(commands).redirectErrorStream(true);

            final Process process = processBuilder.start();

            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
            String line;
            while ((line = bufferedReader.readLine()) != null) {
                lines.add(line);
            }
            // Allow process to run up to 60 seconds
            process.waitFor(60, TimeUnit.SECONDS);
            // Get exit code from process
            exitCode = process.exitValue();
            //Convert exit code to meaningful statement
            String exitMessage = exitCodeToErrorMessage(exitCode);               
            lines.add(exitMessage);
        } catch (Exception ex) {
            logger.error(ex);
        }

        return convertLinesToString(lines);
    }


    private String exitCodeToErrorMessage(Integer exitCode) {
        String errorMessage = null;
        switch (exitCode) {
        case 0:
            errorMessage = "Success.";
            break;
        case 1:
            errorMessage = "Syntax or usage error.";
            break;
        case 2:
            errorMessage = "Protocol incompatibility.";
            break;
        case 3:
            errorMessage = "Errors selecting input/output files, dirs.";
            break;
        case 4:
            errorMessage = "Requested action not supported: an attempt was made to manipulate 64-bit files on a platform that cannot support them; or an option was specified that is supported by the client and not by the server.";
            break;
        case 5:
            errorMessage = "Error starting client-server protocol.";
            break;
        case 6:
            errorMessage = "Daemon unable to append to log-file.";
            break;
        case 10:
            errorMessage = "Error in socket I/O.";
            break;
        case 11:
            errorMessage = "Error in file I/O.";
            break;
        case 12:
            errorMessage = "Error in rsync protocol data stream.";
            break;
        case 13:
            errorMessage = "Errors with program diagnostics.";
            break;
        case 14:
            errorMessage = "Error in IPC code.";
            break;
        case 20:
            errorMessage = "Received SIGUSR1 or SIGINT.";
            break;
        case 21:
            errorMessage = "Some error returned by waitpid().";
            break;
        case 22:
            errorMessage = "Error allocating core memory buffers.";
            break;
        case 23:
            errorMessage = "Partial transfer due to error.";
            break;
        case 24:
            errorMessage = "Partial transfer due to vanished source files.";
            break;
        case 25:
            errorMessage = "The --max-delete limit stopped deletions.";
            break;
        case 30:
            errorMessage = "Timeout in data send/receive.";
            break;
        case 35:
            errorMessage = "Timeout waiting for daemon connection.";
            break;
        default:
            errorMessage = "Unrecognised error code.";
        }
        return errorMessage;
    }

    protected String convertLinesToString(List<String> lines) {
        String result = null;

        if (lines != null && !lines.isEmpty()) {
            StringBuilder builder = new StringBuilder();
            for (String line : lines) {
                builder.append(line).append(" ");
            }
            result = builder.toString().trim();
        }
        return result;
    }

    protected List<String> createCommandLine(String source, String destination) {
        // rsync -rtvuch <source> <destination>
        List<String> commands = new ArrayList<>();

        commands.add("rsync");
        commands.add("-rtvuch");

        String escapedSource = source.trim();
        String escapedDestination = destination.trim();

        commands.add(escapedSource);
        commands.add(escapedDestination);
        logger.debug("escapedSource " + escapedSource);
        logger.debug("escapedDestination " + escapedDestination);

        return commands;
    }

}

Answer 2

Have you seen the answer here?

While there is a bug around quotes managements in Common Exec, this answers suggests:

// When writing a command with space use double "
cmdLine.addArgument(--grep=\"\"" + filter+"\"\"", false"\"\"",false);

Answer 3

I think that Apache Commons Exec has a bug here and cannot properly handle arguments with spaces in them. This open bug (that m4gic linked to) has several contradictory explanations and workarounds: https://issues.apache.org/jira/browse/EXEC-54

I would suggest rewriting your code to use Java's built in "Runtime.exec" function, with the String[] form of the command-line (or the equivalent java.lang.ProcessBuilder API), to bypass any shell escaping and quoting issues entirely.

You will have to manually deal with a few technical details that Apache Commons Exec hides from you, such as potential i/o deadlock between stdout and stderr, but it is quite possible to deal with those whereas I don't think it is possible to get Apache to do what you want here. The Apache Commons Exec intro discusses some of these issues. You should be able to find some example code using Runtime.exec or ProcessBuilder on the internet. GL!