From fab4a0fb4d74fbbfba55944f36544cc53b43364d Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Tue, 6 Apr 2010 15:13:08 -0700 Subject: [PATCH 1/3] tuned runscript to use herefile syntax so that it is easier to troubleshoot and works without needing to escape characters --- .../compute/EC2ComputeServiceLiveTest.java | 4 +- .../jclouds/scriptbuilder/InitBuilder.java | 6 +- .../scriptbuilder/domain/CreateRunScript.java | 96 ++++++++++++++----- .../scriptbuilder/domain/SwitchArg.java | 21 +++- scriptbuilder/src/test/resources/test_ebs.sh | 91 ++++++++++-------- .../src/test/resources/test_init.cmd | 9 +- scriptbuilder/src/test/resources/test_init.sh | 51 ++++++---- .../src/test/resources/test_runrun.sh | 38 +++++--- .../src/test/resources/test_runrun_header.sh | 33 +++++++ 9 files changed, 246 insertions(+), 103 deletions(-) create mode 100644 scriptbuilder/src/test/resources/test_runrun_header.sh diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java index 76076e9099..db86fb5e31 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java @@ -28,6 +28,8 @@ import org.jclouds.ssh.jsch.config.JschSshClientModule; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import com.google.common.collect.Iterables; + /** * * @author Adrian Cole @@ -42,7 +44,7 @@ public class EC2ComputeServiceLiveTest extends BaseComputeServiceLiveTest { @Test public void testTemplateBuilderCanUseImageId() { - client.templateBuilder().imageId("ami-d57f93bc").build(); + client.templateBuilder().imageId(Iterables.get(client.getImages().keySet(), 0)).build(); } @Test diff --git a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java index b346d2caa5..1833c2fd92 100644 --- a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java +++ b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java @@ -89,17 +89,17 @@ public class InitBuilder extends ScriptBuilder { "tail", newStatementList( call("default"), - interpret("tail {varl}LOG_DIR{varr}{fs}stdout.log"))) + interpret("tail {varl}LOG_DIR{varr}{fs}stdout.log{lf}"))) .put( "tailerr", newStatementList( call("default"), - interpret("tail {varl}LOG_DIR{varr}{fs}stderr.log"))) + interpret("tail {varl}LOG_DIR{varr}{fs}stderr.log{lf}"))) .put( "run", newStatementList( call("default"), - interpret("{varl}INSTANCE_HOME{varr}{fs}{varl}INSTANCE_NAME{varr}.{sh}"))) + interpret("{varl}INSTANCE_HOME{varr}{fs}{varl}INSTANCE_NAME{varr}.{sh}{lf}"))) .build())); } } \ No newline at end of file diff --git a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/CreateRunScript.java b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/CreateRunScript.java index 6a6f2b8987..dcbfc806f5 100644 --- a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/CreateRunScript.java +++ b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/CreateRunScript.java @@ -19,6 +19,7 @@ package org.jclouds.scriptbuilder.domain; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static org.jclouds.scriptbuilder.domain.Statements.interpret; import java.util.Collections; @@ -42,6 +43,7 @@ import com.google.common.collect.Maps; * @author Adrian Cole */ public class CreateRunScript implements Statement { + public final static String MARKER = "END_OF_SCRIPT"; final String instanceName; final Iterable exports; final String pwd; @@ -57,6 +59,7 @@ public class CreateRunScript implements Statement { this.exports = checkNotNull(exports, "exports"); this.pwd = checkNotNull(pwd, "pwd").replaceAll("[/\\\\]", "{fs}"); this.execLines = checkNotNull(execLines, "execLines"); + checkState(execLines.length > 0, "you must pass something to execute"); } public static class AddTitleToFile implements Statement { @@ -142,36 +145,81 @@ public class CreateRunScript implements Statement { Map tokenMap = ShellToken.tokenValueMap(family); String runScript = Utils.replaceTokens(pwd + "{fs}" + instanceName + ".{sh}", tokenMap); statements.add(interpret(String.format("{md} %s{lf}", pwd))); - statements.add(interpret(String.format("{rm} %s 2{closeFd}{lf}", runScript))); - for (String line : Splitter.on(ShellToken.LF.to(family)).split( - ShellToken.BEGIN_SCRIPT.to(family))) { - if (!line.equals("")) - statements.add(appendToFile(line, runScript, family)); - } - statements.add(new AddTitleToFile(instanceName, runScript)); - statements - .add(appendToFile(Utils.writeZeroPath(family).replace(ShellToken.LF.to(family), ""), - runScript, family)); - statements.add(new AddExportToFile("instanceName", instanceName, runScript)); - for (String export : exports) { - statements.add(new AddExportToFile(export, Utils.replaceTokens("{varl}" - + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, export) + "{varr}", - tokenMap), runScript)); - } - statements.add(appendToFile("{cd} " + pwd, runScript, family)); - for (String execLine : execLines) { - statements.add(appendToFile(execLine, runScript, family)); - } - for (String line : Splitter.on(ShellToken.LF.to(family)).split( - ShellToken.END_SCRIPT.to(family))) { - if (!line.equals("")) - statements.add(appendToFile(line, runScript, family)); + if (family == OsFamily.UNIX) { + StringBuilder builder = new StringBuilder(); + builder.append("\n"); + addUnixRunScriptHeader(family, runScript, builder); + builder.append("\n"); + addUnixRunScript(runScript, builder); + builder.append("\n"); + addUnixRunScriptFooter(family, runScript, builder); + builder.append("\n"); + statements.add(interpret(builder.toString())); + } else { + statements.add(interpret(String.format("{rm} %s 2{closeFd}{lf}", runScript))); + for (String line : Splitter.on(ShellToken.LF.to(family)).split( + ShellToken.BEGIN_SCRIPT.to(family))) { + if (!line.equals("")) + statements.add(appendToFile(line, runScript, family)); + } + statements.add(new AddTitleToFile(instanceName, runScript)); + statements.add(appendToFile(Utils.writeZeroPath(family).replace(ShellToken.LF.to(family), + ""), runScript, family)); + statements.add(new AddExportToFile("instanceName", instanceName, runScript)); + for (String export : exports) { + statements.add(new AddExportToFile(export, Utils.replaceTokens("{varl}" + + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, export) + "{varr}", + tokenMap), runScript)); + } + statements.add(appendToFile("{cd} " + pwd, runScript, family)); + for (String execLine : execLines) { + statements.add(appendToFile(execLine, runScript, family)); + } + for (String line : Splitter.on(ShellToken.LF.to(family)).split( + ShellToken.END_SCRIPT.to(family))) { + if (!line.equals("")) + statements.add(appendToFile(line, runScript, family)); + } } statements.add(interpret(Utils.replaceTokens(OS_TO_CHMOD_PATTERN.get(family), ImmutableMap .of("file", runScript)))); return new StatementList(statements).render(family); } + private void addUnixRunScriptFooter(OsFamily family, String runScript, StringBuilder builder) { + builder.append("# add runscript footer\n"); + builder.append("cat >> ").append(runScript).append(" <<'").append(MARKER).append("'\n"); + builder.append(ShellToken.END_SCRIPT.to(family)); + builder.append(MARKER).append("\n"); + } + + private void addUnixRunScript(String runScript, StringBuilder builder) { + builder.append("# add desired commands from the user\n"); + builder.append("cat >> ").append(runScript).append(" <<'").append(MARKER).append("'\n"); + builder.append("cd ").append(pwd).append("\n"); + for (String execLine : execLines) { + builder.append(execLine).append("\n"); + } + builder.append(MARKER).append("\n"); + } + + private void addUnixRunScriptHeader(OsFamily family, String runScript, StringBuilder builder) { + builder.append("# create runscript header\n"); + builder.append("cat > ").append(runScript).append(" <<").append(MARKER).append("\n"); + builder.append(ShellToken.BEGIN_SCRIPT.to(family)); + builder.append("PROMPT_COMMAND='echo -ne \"\\033]0;").append(instanceName).append( + "\\007\"'\n"); + builder.append(Utils.writeZeroPath(family)); + builder.append("export INSTANCE_NAME='").append(instanceName).append("'\n"); + for (String export : exports) { + String variableNameInUpper = CaseFormat.LOWER_CAMEL + .to(CaseFormat.UPPER_UNDERSCORE, export); + builder.append("export ").append(variableNameInUpper).append("='$").append( + variableNameInUpper).append("'\n"); + } + builder.append(MARKER).append("\n"); + } + private Statement appendToFile(String line, String runScript, OsFamily family) { String quote = ""; if (!ShellToken.VQ.to(family).equals("")) { diff --git a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/SwitchArg.java b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/SwitchArg.java index 62b9b8b131..4f2342f2c5 100644 --- a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/SwitchArg.java +++ b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/domain/SwitchArg.java @@ -28,6 +28,7 @@ import org.jclouds.scriptbuilder.util.Utils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -39,6 +40,8 @@ import com.google.common.collect.Lists; */ public class SwitchArg implements Statement { + private static final String INDENT = " "; + public static final Map OS_TO_SWITCH_PATTERN = ImmutableMap.of(OsFamily.UNIX, "case ${arg} in\n", OsFamily.WINDOWS, "goto CASE_%{arg}\r\n"); @@ -84,10 +87,22 @@ public class SwitchArg implements Statement { "arg", arg + ""))); for (Entry entry : valueToActions.entrySet()) { + + StringBuilder actionBuilder = new StringBuilder(); + boolean inRunScript = false; + for (String line : Splitter.on(ShellToken.LF.to(family)).split( + entry.getValue().render(family))) { + if (!inRunScript) + actionBuilder.append(INDENT); + actionBuilder.append(line).append(ShellToken.LF.to(family)); + if (line.indexOf(CreateRunScript.MARKER) != -1) { + inRunScript = inRunScript ? false : true; + } + } + actionBuilder.delete(actionBuilder.lastIndexOf(ShellToken.LF.to(family)), actionBuilder + .length()); switchClause.append(Utils.replaceTokens(OS_TO_CASE_PATTERN.get(family), ImmutableMap.of( - "value", entry.getKey(), "action", entry.getValue().render(family).replaceAll( - "^", " ").replace(ShellToken.LF.to(family), - ShellToken.LF.to(family) + " ")))); + "value", entry.getKey(), "action", actionBuilder.toString()))); } switchClause.append(OS_TO_END_SWITCH_PATTERN.get(family)); diff --git a/scriptbuilder/src/test/resources/test_ebs.sh b/scriptbuilder/src/test/resources/test_ebs.sh index d5c72ad132..69f9b33914 100755 --- a/scriptbuilder/src/test/resources/test_ebs.sh +++ b/scriptbuilder/src/test/resources/test_ebs.sh @@ -61,41 +61,53 @@ init) default || exit 1 mkebsboot || exit 1 mkdir -p $INSTANCE_HOME - rm $INSTANCE_HOME/mkebsboot.sh 2>&- - echo '#!/bin/bash'>>$INSTANCE_HOME/mkebsboot.sh - echo 'set +u'>>$INSTANCE_HOME/mkebsboot.sh - echo 'shopt -s xpg_echo'>>$INSTANCE_HOME/mkebsboot.sh - echo 'shopt -s expand_aliases'>>$INSTANCE_HOME/mkebsboot.sh - echo "PROMPT_COMMAND='echo -ne \"\033]0;mkebsboot\007\"'">>$INSTANCE_HOME/mkebsboot.sh - echo 'export PATH=/usr/ucb/bin:/bin:/sbin:/usr/bin:/usr/sbin'>>$INSTANCE_HOME/mkebsboot.sh - echo "export INSTANCE_NAME='mkebsboot'">>$INSTANCE_HOME/mkebsboot.sh - echo "export IMAGE_DIR='$IMAGE_DIR'">>$INSTANCE_HOME/mkebsboot.sh - echo "export EBS_DEVICE='$EBS_DEVICE'">>$INSTANCE_HOME/mkebsboot.sh - echo "export EBS_MOUNT_POINT='$EBS_MOUNT_POINT'">>$INSTANCE_HOME/mkebsboot.sh - echo "export INSTANCE_NAME='$INSTANCE_NAME'">>$INSTANCE_HOME/mkebsboot.sh - echo "export INSTANCE_HOME='$INSTANCE_HOME'">>$INSTANCE_HOME/mkebsboot.sh - echo "export LOG_DIR='$LOG_DIR'">>$INSTANCE_HOME/mkebsboot.sh - echo 'cd $INSTANCE_HOME'>>$INSTANCE_HOME/mkebsboot.sh - echo 'echo creating a filesystem and mounting the ebs volume'>>$INSTANCE_HOME/mkebsboot.sh - echo 'mkdir -p $IMAGE_DIR $EBS_MOUNT_POINT'>>$INSTANCE_HOME/mkebsboot.sh - echo 'rm -rf $IMAGE_DIR/*'>>$INSTANCE_HOME/mkebsboot.sh - echo 'yes| mkfs -t ext3 $EBS_DEVICE 2>&-'>>$INSTANCE_HOME/mkebsboot.sh - echo 'mount $EBS_DEVICE $EBS_MOUNT_POINT'>>$INSTANCE_HOME/mkebsboot.sh - echo 'echo making a local working copy of the boot disk'>>$INSTANCE_HOME/mkebsboot.sh - echo 'rsync -ax --exclude /ubuntu/.bash_history --exclude /home/*/.bash_history --exclude /etc/ssh/ssh_host_* --exclude /etc/ssh/moduli --exclude /etc/udev/rules.d/*persistent-net.rules --exclude /var/lib/ec2/* --exclude=/mnt/* --exclude=/proc/* --exclude=/tmp/* --exclude=/dev/log / $IMAGE_DIR'>>$INSTANCE_HOME/mkebsboot.sh - echo 'echo preparing the local working copy'>>$INSTANCE_HOME/mkebsboot.sh - echo 'touch $IMAGE_DIR/etc/init.d/ec2-init-user-data'>>$INSTANCE_HOME/mkebsboot.sh - echo 'echo copying the local working copy to the ebs mount'>>$INSTANCE_HOME/mkebsboot.sh - echo 'cd $IMAGE_DIR'>>$INSTANCE_HOME/mkebsboot.sh - echo 'tar -cSf - * | tar xf - -C $EBS_MOUNT_POINT'>>$INSTANCE_HOME/mkebsboot.sh - echo 'echo size of ebs'>>$INSTANCE_HOME/mkebsboot.sh - echo 'du -sk $EBS_MOUNT_POINT'>>$INSTANCE_HOME/mkebsboot.sh - echo 'echo size of source'>>$INSTANCE_HOME/mkebsboot.sh - echo 'du -sk $IMAGE_DIR'>>$INSTANCE_HOME/mkebsboot.sh - echo 'rm -rf $IMAGE_DIR/*'>>$INSTANCE_HOME/mkebsboot.sh - echo 'umount $EBS_MOUNT_POINT'>>$INSTANCE_HOME/mkebsboot.sh - echo 'echo ----COMPLETE----'>>$INSTANCE_HOME/mkebsboot.sh - echo 'exit 0'>>$INSTANCE_HOME/mkebsboot.sh + + # create runscript header + cat > $INSTANCE_HOME/mkebsboot.sh <> $INSTANCE_HOME/mkebsboot.sh <<'END_OF_SCRIPT' +cd $INSTANCE_HOME +echo creating a filesystem and mounting the ebs volume +mkdir -p $IMAGE_DIR $EBS_MOUNT_POINT +rm -rf $IMAGE_DIR/* +yes| mkfs -t ext3 $EBS_DEVICE 2>&- +mount $EBS_DEVICE $EBS_MOUNT_POINT +echo making a local working copy of the boot disk +rsync -ax --exclude /ubuntu/.bash_history --exclude /home/*/.bash_history --exclude /etc/ssh/ssh_host_* --exclude /etc/ssh/moduli --exclude /etc/udev/rules.d/*persistent-net.rules --exclude /var/lib/ec2/* --exclude=/mnt/* --exclude=/proc/* --exclude=/tmp/* --exclude=/dev/log / $IMAGE_DIR +echo preparing the local working copy +touch $IMAGE_DIR/etc/init.d/ec2-init-user-data +echo copying the local working copy to the ebs mount +cd $IMAGE_DIR +tar -cSf - * | tar xf - -C $EBS_MOUNT_POINT +echo size of ebs +du -sk $EBS_MOUNT_POINT +echo size of source +du -sk $IMAGE_DIR +rm -rf $IMAGE_DIR/* +umount $EBS_MOUNT_POINT +echo ----COMPLETE---- +END_OF_SCRIPT + + # add runscript footer + cat >> $INSTANCE_HOME/mkebsboot.sh <<'END_OF_SCRIPT' +exit 0 +END_OF_SCRIPT + chmod u+x $INSTANCE_HOME/mkebsboot.sh ;; status) @@ -117,12 +129,15 @@ start) ;; tail) default || exit 1 - tail $LOG_DIR/stdout.log;; + tail $LOG_DIR/stdout.log + ;; tailerr) default || exit 1 - tail $LOG_DIR/stderr.log;; + tail $LOG_DIR/stderr.log + ;; run) default || exit 1 - $INSTANCE_HOME/$INSTANCE_NAME.sh;; + $INSTANCE_HOME/$INSTANCE_NAME.sh + ;; esac exit 0 diff --git a/scriptbuilder/src/test/resources/test_init.cmd b/scriptbuilder/src/test/resources/test_init.cmd index 1f5838b9a7..e0d4370967 100644 --- a/scriptbuilder/src/test/resources/test_init.cmd +++ b/scriptbuilder/src/test/resources/test_init.cmd @@ -108,14 +108,17 @@ goto CASE_%1 :CASE_tail call :default if errorlevel 1 goto abort - tail %LOG_DIR%\stdout.logGOTO END_SWITCH + tail %LOG_DIR%\stdout.log + GOTO END_SWITCH :CASE_tailerr call :default if errorlevel 1 goto abort - tail %LOG_DIR%\stderr.logGOTO END_SWITCH + tail %LOG_DIR%\stderr.log + GOTO END_SWITCH :CASE_run call :default if errorlevel 1 goto abort - %INSTANCE_HOME%\%INSTANCE_NAME%.cmdGOTO END_SWITCH + %INSTANCE_HOME%\%INSTANCE_NAME%.cmd + GOTO END_SWITCH :END_SWITCH exit /b 0 diff --git a/scriptbuilder/src/test/resources/test_init.sh b/scriptbuilder/src/test/resources/test_init.sh index 030a63e9e8..c0f768f205 100755 --- a/scriptbuilder/src/test/resources/test_init.sh +++ b/scriptbuilder/src/test/resources/test_init.sh @@ -59,21 +59,33 @@ init) default || exit 1 mkebsboot || exit 1 mkdir -p $INSTANCE_HOME - rm $INSTANCE_HOME/mkebsboot.sh 2>&- - echo '#!/bin/bash'>>$INSTANCE_HOME/mkebsboot.sh - echo 'set +u'>>$INSTANCE_HOME/mkebsboot.sh - echo 'shopt -s xpg_echo'>>$INSTANCE_HOME/mkebsboot.sh - echo 'shopt -s expand_aliases'>>$INSTANCE_HOME/mkebsboot.sh - echo "PROMPT_COMMAND='echo -ne \"\033]0;mkebsboot\007\"'">>$INSTANCE_HOME/mkebsboot.sh - echo 'export PATH=/usr/ucb/bin:/bin:/sbin:/usr/bin:/usr/sbin'>>$INSTANCE_HOME/mkebsboot.sh - echo "export INSTANCE_NAME='mkebsboot'">>$INSTANCE_HOME/mkebsboot.sh - echo "export TMP_DIR='$TMP_DIR'">>$INSTANCE_HOME/mkebsboot.sh - echo "export INSTANCE_NAME='$INSTANCE_NAME'">>$INSTANCE_HOME/mkebsboot.sh - echo "export INSTANCE_HOME='$INSTANCE_HOME'">>$INSTANCE_HOME/mkebsboot.sh - echo "export LOG_DIR='$LOG_DIR'">>$INSTANCE_HOME/mkebsboot.sh - echo 'cd $INSTANCE_HOME'>>$INSTANCE_HOME/mkebsboot.sh - echo 'find /'>>$INSTANCE_HOME/mkebsboot.sh - echo 'exit 0'>>$INSTANCE_HOME/mkebsboot.sh + + # create runscript header + cat > $INSTANCE_HOME/mkebsboot.sh <> $INSTANCE_HOME/mkebsboot.sh <<'END_OF_SCRIPT' +cd $INSTANCE_HOME +find / +END_OF_SCRIPT + + # add runscript footer + cat >> $INSTANCE_HOME/mkebsboot.sh <<'END_OF_SCRIPT' +exit 0 +END_OF_SCRIPT + chmod u+x $INSTANCE_HOME/mkebsboot.sh ;; status) @@ -95,12 +107,15 @@ start) ;; tail) default || exit 1 - tail $LOG_DIR/stdout.log;; + tail $LOG_DIR/stdout.log + ;; tailerr) default || exit 1 - tail $LOG_DIR/stderr.log;; + tail $LOG_DIR/stderr.log + ;; run) default || exit 1 - $INSTANCE_HOME/$INSTANCE_NAME.sh;; + $INSTANCE_HOME/$INSTANCE_NAME.sh + ;; esac exit 0 diff --git a/scriptbuilder/src/test/resources/test_runrun.sh b/scriptbuilder/src/test/resources/test_runrun.sh index af4a9a588f..cad630c83c 100644 --- a/scriptbuilder/src/test/resources/test_runrun.sh +++ b/scriptbuilder/src/test/resources/test_runrun.sh @@ -1,15 +1,27 @@ mkdir -p /tmp/$USER/scripttest -rm /tmp/$USER/scripttest/yahooprod.sh 2>&- -echo '#!/bin/bash'>>/tmp/$USER/scripttest/yahooprod.sh -echo 'set +u'>>/tmp/$USER/scripttest/yahooprod.sh -echo 'shopt -s xpg_echo'>>/tmp/$USER/scripttest/yahooprod.sh -echo 'shopt -s expand_aliases'>>/tmp/$USER/scripttest/yahooprod.sh -echo "PROMPT_COMMAND='echo -ne \"\033]0;yahooprod\007\"'">>/tmp/$USER/scripttest/yahooprod.sh -echo 'export PATH=/usr/ucb/bin:/bin:/sbin:/usr/bin:/usr/sbin'>>/tmp/$USER/scripttest/yahooprod.sh -echo "export INSTANCE_NAME='yahooprod'">>/tmp/$USER/scripttest/yahooprod.sh -echo "export JAVA_HOME='$JAVA_HOME'">>/tmp/$USER/scripttest/yahooprod.sh -echo 'cd /tmp/$USER/scripttest'>>/tmp/$USER/scripttest/yahooprod.sh -echo 'echo hello'>>/tmp/$USER/scripttest/yahooprod.sh -echo 'echo $JAVA_HOME/bin/java -DinstanceName=$INSTANCE_NAME myServer.Main'>>/tmp/$USER/scripttest/yahooprod.sh -echo 'exit 0'>>/tmp/$USER/scripttest/yahooprod.sh + +# create runscript header +cat > /tmp/$USER/scripttest/yahooprod.sh <> /tmp/$USER/scripttest/yahooprod.sh <<'END_OF_SCRIPT' +cd /tmp/$USER/scripttest +echo hello +echo $JAVA_HOME/bin/java -DinstanceName=$INSTANCE_NAME myServer.Main +END_OF_SCRIPT + +# add runscript footer +cat >> /tmp/$USER/scripttest/yahooprod.sh <<'END_OF_SCRIPT' +exit 0 +END_OF_SCRIPT + chmod u+x /tmp/$USER/scripttest/yahooprod.sh diff --git a/scriptbuilder/src/test/resources/test_runrun_header.sh b/scriptbuilder/src/test/resources/test_runrun_header.sh new file mode 100644 index 0000000000..77a633ac1c --- /dev/null +++ b/scriptbuilder/src/test/resources/test_runrun_header.sh @@ -0,0 +1,33 @@ +mkdir -p /tmp/$USER/scripttest + +# create runscript header +( +cat < /tmp/$USER/scripttest/yahooprod.sh + +# add desired commands from the user +( +cat <<'END_OF_SCRIPT' +cd /tmp/$USER/scripttest +echo hello +echo $JAVA_HOME/bin/java -DinstanceName=$INSTANCE_NAME myServer.Main +END_OF_SCRIPT +) >> /tmp/$USER/scripttest/yahooprod.sh + +# add runscript footer +( +cat <<'END_OF_SCRIPT' +exit 0 +END_OF_SCRIPT +) >> /tmp/$USER/scripttest/yahooprod.sh + +chmod u+x /tmp/$USER/scripttest/yahooprod.sh From 45384de903e89edbb608e24fdc556abb21cbbede Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Tue, 6 Apr 2010 19:33:26 -0700 Subject: [PATCH 2/3] Issue 202: with-details option for blobstore list --- blobstore/src/main/clojure/org/jclouds/blobstore.clj | 1 + blobstore/src/test/clojure/org/jclouds/blobstore_test.clj | 1 + 2 files changed, 2 insertions(+) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index c8a353c673..de13cb6bac 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -85,6 +85,7 @@ Options can also be specified for extension modules {:after-marker #(.afterMarker %1 %2) :in-directory #(.inDirectory %1 %2) :max-results #(.maxResults %1 %2) + :with-details #(when %2 (.withDetails %1)) :recursive #(when %2 (.recursive %1))}) (defn list-container diff --git a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj index 424fe024c1..b764890fa6 100644 --- a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj +++ b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj @@ -48,6 +48,7 @@ (is (upload-blob "container" "dir/blob2" "blob2")) (is (= 3 (count (list-container "container")))) (is (= 4 (count (list-container "container" :recursive true)))) + (is (= 3 (count (list-container "container" :with-details true)))) (is (= 1 (count (list-container "container" :in-directory "dir"))))) (deftest download-blob-test From 89958f93d13c927ceeffdeb46d799194d70450d7 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 7 Apr 2010 12:47:49 -0700 Subject: [PATCH 3/3] normalized image parsing across canonical and alestic; added tests to kickout non-amis and test images --- .../jclouds/aws/ec2/EC2PropertiesBuilder.java | 3 +- .../EC2ComputeServiceContextModule.java | 8 +- .../ec2/compute/functions/ImageParser.java | 32 ++++++-- .../xml/DescribeImagesResponseHandler.java | 2 +- .../compute/EC2ComputeServiceLiveTest.java | 6 ++ .../aws/ec2/functions/ImageParserTest.java | 77 ++++++++++++++++- .../test/resources/ec2/alestic_canonical.xml | 82 +++++++++++++++++++ 7 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 aws/core/src/test/resources/ec2/alestic_canonical.xml diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/EC2PropertiesBuilder.java b/aws/core/src/main/java/org/jclouds/aws/ec2/EC2PropertiesBuilder.java index f1ef97d7c0..4613e15b61 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/EC2PropertiesBuilder.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/EC2PropertiesBuilder.java @@ -41,7 +41,8 @@ public class EC2PropertiesBuilder extends PropertiesBuilder { Properties properties = super.defaultProperties(); properties.setProperty(PROPERTY_EC2_ENDPOINT, "https://ec2.us-east-1.amazonaws.com"); properties.setProperty(PROPERTY_AWS_EXPIREINTERVAL, "60"); - properties.setProperty(PROPERTY_EC2_AMI_OWNERS, "063491364108"); + // alestic and canonical + properties.setProperty(PROPERTY_EC2_AMI_OWNERS, "063491364108,099720109477"); return properties; } diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java index 3269eea0a1..ced9332edb 100755 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java @@ -55,6 +55,7 @@ import org.jclouds.aws.ec2.domain.RunningInstance; import org.jclouds.aws.ec2.services.InstanceClient; import org.jclouds.compute.ComputeService; import org.jclouds.compute.ComputeServiceContext; +import org.jclouds.compute.domain.Architecture; import org.jclouds.compute.domain.ComputeMetadata; import org.jclouds.compute.domain.Image; import org.jclouds.compute.domain.NodeMetadata; @@ -109,7 +110,8 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule { TemplateBuilder provideTemplate(Map locations, Map images, Map sizes, Location defaultLocation) { - return new TemplateBuilderImpl(locations, images, sizes, defaultLocation).osFamily(UBUNTU); + return new TemplateBuilderImpl(locations, images, sizes, defaultLocation).architecture( + Architecture.X86_32).osFamily(UBUNTU); } @Singleton @@ -291,7 +293,7 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule { @Singleton @Named(PROPERTY_EC2_AMI_OWNERS) String[] amiOwners(@Named(PROPERTY_EC2_AMI_OWNERS) String amiOwners) { - return Iterables.toArray(Splitter.on('.').split(amiOwners), String.class); + return Iterables.toArray(Splitter.on(',').split(amiOwners), String.class); } @Provides @@ -311,7 +313,7 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule { if (image != null) images.add(image); else - holder.logger.debug("<< images(%s) didn't parse", from.getId()); + holder.logger.trace("<< image(%s) didn't parse", from.getId()); } } holder.logger.debug("<< images(%d)", images.size()); diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/ImageParser.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/ImageParser.java index fa792f0c2e..e1e450ea32 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/ImageParser.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/ImageParser.java @@ -18,6 +18,7 @@ */ package org.jclouds.aws.ec2.compute.functions; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,6 +26,7 @@ import javax.annotation.Resource; import javax.inject.Named; import javax.inject.Singleton; +import org.jclouds.aws.ec2.domain.Image.ImageType; import org.jclouds.compute.domain.Architecture; import org.jclouds.compute.domain.Image; import org.jclouds.compute.domain.OsFamily; @@ -35,32 +37,48 @@ import org.jclouds.logging.Logger; import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; +/** + * @author Adrian Cole + */ @Singleton -public class ImageParser implements - Function { +public class ImageParser implements Function { @Resource @Named(ComputeServiceConstants.COMPUTE_LOGGER) protected Logger logger = Logger.NULL; - // alestic-32-eu-west-1/debian-4.0-etch-base-20081130 - public static final Pattern ALESTIC_PATTERN = Pattern + public static final Pattern CANONICAL_PATTERN = Pattern .compile(".*/([^-]*)-([^-]*)-.*-(.*)(\\.manifest.xml)?"); + public static final Map NAME_VERSION_MAP = ImmutableMap + . builder().put("hardy", "8.04").put("intrepid", "8.10").put("jaunty", + "9.04").put("karmic", "9.10").put("lucid", "10.04").put("maverick", "10.10") + .build(); + @Override public Image apply(org.jclouds.aws.ec2.domain.Image from) { + if (from.getImageLocation().indexOf("test") != -1) { + logger.trace("skipping test image(%s)", from.getId()); + return null; + } + if (from.getImageType() != ImageType.MACHINE) { + logger.trace("skipping as not a machine image(%s)", from.getId()); + return null; + } OsFamily os = null; String name = ""; String description = from.getDescription() != null ? from.getDescription() : from .getImageLocation(); String osDescription = from.getImageLocation(); String version = ""; - - Matcher matcher = ALESTIC_PATTERN.matcher(from.getImageLocation()); + Matcher matcher = CANONICAL_PATTERN.matcher(from.getImageLocation()); if (matcher.find()) { try { os = OsFamily.fromValue(matcher.group(1)); name = matcher.group(2);// TODO no field for os version - version = matcher.group(3).replace("\\.manifest.xml", ""); + // normalize versions across ubuntu from alestic and canonical + if (NAME_VERSION_MAP.containsKey(name)) + name = NAME_VERSION_MAP.get(name); + version = matcher.group(3).replace(".manifest.xml", ""); } catch (IllegalArgumentException e) { logger.debug("<< didn't match os(%s)", matcher.group(1)); } diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeImagesResponseHandler.java b/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeImagesResponseHandler.java index 162f0d09a8..f9588f7c46 100755 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeImagesResponseHandler.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeImagesResponseHandler.java @@ -50,7 +50,7 @@ public class DescribeImagesResponseHandler extends ParseSax.HandlerWithResult contents = Sets.newHashSet(); + private Set contents = Sets.newLinkedHashSet(); private StringBuilder currentText = new StringBuilder(); private Architecture architecture; diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java index db86fb5e31..57676a17be 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java @@ -50,10 +50,16 @@ public class EC2ComputeServiceLiveTest extends BaseComputeServiceLiveTest { @Test public void testTemplateBuilder() { Template defaultTemplate = client.templateBuilder().build(); + assert (defaultTemplate.getImage().getId().startsWith("ami-")) : defaultTemplate; + assertEquals(defaultTemplate.getImage().getName(), "9.10"); assertEquals(defaultTemplate.getImage().getArchitecture(), Architecture.X86_32); assertEquals(defaultTemplate.getImage().getOsFamily(), OsFamily.UBUNTU); assertEquals(defaultTemplate.getLocation().getId(), "us-east-1"); assertEquals(defaultTemplate.getSize().getCores(), 1.0d); + client.templateBuilder().osFamily(OsFamily.UBUNTU).smallest().architecture( + Architecture.X86_32).imageId("ami-7e28ca17").build(); + client.templateBuilder().osFamily(OsFamily.UBUNTU).smallest().architecture( + Architecture.X86_32).imageId("ami-bb709dd2").build(); } @Override diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/functions/ImageParserTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/functions/ImageParserTest.java index d4f3e09748..d576881f4e 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/functions/ImageParserTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/functions/ImageParserTest.java @@ -30,6 +30,7 @@ import org.jclouds.aws.domain.Region; import org.jclouds.aws.ec2.compute.functions.ImageParser; import org.jclouds.aws.ec2.domain.Image; import org.jclouds.aws.ec2.xml.DescribeImagesResponseHandler; +import org.jclouds.compute.domain.OsFamily; import org.jclouds.http.functions.BaseHandlerTest; import org.jclouds.http.functions.ParseSax; import org.jclouds.rest.internal.GeneratedHttpRequest; @@ -44,6 +45,80 @@ import com.google.common.collect.Iterables; @Test(groups = "unit", testName = "compute.PropertiesTest") public class ImageParserTest extends BaseHandlerTest { + public void testParseAlesticCanonicalImage() { + InputStream is = getClass().getResourceAsStream("/ec2/alestic_canonical.xml"); + + Set result = parseImages(is); + assertEquals(result.size(), 6); + + ImageParser parser = new ImageParser(); + org.jclouds.compute.domain.Image ubuntuHardy = parser.apply(Iterables.get(result, 0)); + + assertEquals(ubuntuHardy.getArchitecture(), org.jclouds.compute.domain.Architecture.X86_32); + assertEquals(ubuntuHardy.getDescription(), + "ubuntu-images-us/ubuntu-hardy-8.04-i386-server-20091130.manifest.xml"); + assertEquals(ubuntuHardy.getId(), "ami-7e28ca17"); + assertEquals(ubuntuHardy.getLocationId(), "default"); + assertEquals(ubuntuHardy.getName(), "8.04"); + assertEquals(ubuntuHardy.getOsDescription(), + "ubuntu-images-us/ubuntu-hardy-8.04-i386-server-20091130.manifest.xml"); + assertEquals(ubuntuHardy.getOsFamily(), OsFamily.UBUNTU); + assertEquals(ubuntuHardy.getUserMetadata(), ImmutableMap. of("owner", + "099720109477")); + assertEquals(ubuntuHardy.getVersion(), "20091130"); + + org.jclouds.compute.domain.Image alesticKarmic = parser.apply(Iterables.get(result, 1)); + + assertEquals(alesticKarmic.getArchitecture(), org.jclouds.compute.domain.Architecture.X86_32); + assertEquals(alesticKarmic.getDescription(), + "alestic/ubuntu-9.10-karmic-base-20090623.manifest.xml"); + assertEquals(alesticKarmic.getId(), "ami-19a34270"); + assertEquals(alesticKarmic.getLocationId(), "default"); + assertEquals(alesticKarmic.getName(), "9.10"); + assertEquals(alesticKarmic.getOsDescription(), + "alestic/ubuntu-9.10-karmic-base-20090623.manifest.xml"); + assertEquals(alesticKarmic.getOsFamily(), OsFamily.UBUNTU); + assertEquals(alesticKarmic.getUserMetadata(), ImmutableMap. of("owner", + "063491364108")); + assertEquals(alesticKarmic.getVersion(), "20090623"); + + org.jclouds.compute.domain.Image ubuntuKarmic = parser.apply(Iterables.get(result, 2)); + + assertEquals(ubuntuKarmic.getArchitecture(), org.jclouds.compute.domain.Architecture.X86_32); + assertEquals(ubuntuKarmic.getDescription(), + "ubuntu-images-us/ubuntu-karmic-9.10-i386-server-20100121.manifest.xml"); + assertEquals(ubuntuKarmic.getId(), "ami-bb709dd2"); + assertEquals(ubuntuKarmic.getLocationId(), "default"); + assertEquals(ubuntuKarmic.getName(), "9.10"); + assertEquals(ubuntuKarmic.getOsDescription(), + "ubuntu-images-us/ubuntu-karmic-9.10-i386-server-20100121.manifest.xml"); + assertEquals(ubuntuKarmic.getOsFamily(), OsFamily.UBUNTU); + assertEquals(ubuntuKarmic.getUserMetadata(), ImmutableMap. of("owner", + "099720109477")); + assertEquals(ubuntuKarmic.getVersion(), "20100121"); + + // should skip testing image + assert parser.apply(Iterables.get(result, 3)) == null; + + org.jclouds.compute.domain.Image alesticHardy = parser.apply(Iterables.get(result, 4)); + + assertEquals(alesticHardy.getArchitecture(), org.jclouds.compute.domain.Architecture.X86_32); + assertEquals(alesticHardy.getDescription(), + "alestic/ubuntu-8.04-hardy-base-20080905.manifest.xml"); + assertEquals(alesticHardy.getId(), "ami-c0fa1ea9"); + assertEquals(alesticHardy.getLocationId(), "default"); + assertEquals(alesticHardy.getName(), "8.04"); + assertEquals(alesticHardy.getOsDescription(), + "alestic/ubuntu-8.04-hardy-base-20080905.manifest.xml"); + assertEquals(alesticHardy.getOsFamily(), OsFamily.UBUNTU); + assertEquals(alesticHardy.getUserMetadata(), ImmutableMap. of("owner", + "063491364108")); + assertEquals(alesticHardy.getVersion(), "20080905"); + + // should skip kernel + assert parser.apply(Iterables.get(result, 5)) == null; + } + public void testParseVostokImage() { InputStream is = getClass().getResourceAsStream("/ec2/vostok.xml"); @@ -78,7 +153,7 @@ public class ImageParserTest extends BaseHandlerTest { private void addDefaultRegionToHandler(ParseSax.HandlerWithResult handler) { GeneratedHttpRequest request = createMock(GeneratedHttpRequest.class); - expect(request.getArgs()).andReturn(new Object[] { Region.DEFAULT }); + expect(request.getArgs()).andReturn(new Object[] { Region.DEFAULT }).atLeastOnce(); replay(request); handler.setContext(request); } diff --git a/aws/core/src/test/resources/ec2/alestic_canonical.xml b/aws/core/src/test/resources/ec2/alestic_canonical.xml new file mode 100644 index 0000000000..40f6ecff9c --- /dev/null +++ b/aws/core/src/test/resources/ec2/alestic_canonical.xml @@ -0,0 +1,82 @@ + + + 6104eee1-affd-49d7-92a0-516cab8a8ba6 + + + ami-7e28ca17 + ubuntu-images-us/ubuntu-hardy-8.04-i386-server-20091130.manifest.xml + available + 099720109477 + true + i386 + machine + aki-92ba58fb + ari-94ba58fd + instance-store + + + + ami-19a34270 + alestic/ubuntu-9.10-karmic-base-20090623.manifest.xml + available + 063491364108 + true + i386 + machine + aki-a71cf9ce + ari-a51cf9cc + instance-store + + + + ami-bb709dd2 + ubuntu-images-us/ubuntu-karmic-9.10-i386-server-20100121.manifest.xml + available + 099720109477 + true + i386 + machine + aki-5f15f636 + ari-d5709dbc + instance-store + + + + ami-190fe070 + ubuntu-images-testing-us/ubuntu-lucid-daily-amd64-desktop-20100317.manifest.xml + available + 099720109477 + true + x86_64 + machine + aki-11c72878 + ubuntu-images-testing/ubuntu-lucid-daily-amd64-desktop-20100317 + instance-store + + + + ami-c0fa1ea9 + alestic/ubuntu-8.04-hardy-base-20080905.manifest.xml + available + 063491364108 + true + i386 + machine + aki-a71cf9ce + ari-a51cf9cc + instance-store + + + + aki-fd15f694 + ubuntu-kernels-us/ubuntu-karmic-amd64-linux-image-2.6.31-302-ec2-v-2.6.31-302.7-kernel.img.manifest.xml + available + 099720109477 + true + x86_64 + kernel + instance-store + + + +