Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove tar creation responsibility from docker, avoiding filesystem bias.

This commit is contained in:
Adrian Cole 2014-10-28 16:51:19 -07:00 committed by Adrian Cole
parent 7bdff3f9ca
commit ef59a121bf
9 changed files with 43 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apis/docker/pom.xml
View File

@ -58,11 +58,6 @@
<artifactId>jclouds-compute</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.5</version>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
@ -112,6 +107,13 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jboss.shrinkwrap</groupId>
<artifactId>shrinkwrap-depchain</artifactId>
<version>1.2.2</version>
<type>pom</type>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>

68
apis/docker/src/main/java/org/jclouds/docker/binders/BindInputStreamToRequest.java
View File

@ -1,68 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.docker.binders;
import com.google.common.base.Throwables;
import com.google.common.io.Files;
import org.jclouds.compute.reference.ComputeServiceConstants;
import org.jclouds.docker.features.internal.Archives;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.logging.Logger;
import org.jclouds.rest.Binder;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
@Singleton
public class BindInputStreamToRequest implements Binder {
@Resource
@Named(ComputeServiceConstants.COMPUTE_LOGGER)
protected Logger logger = Logger.NULL;
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object input) {
checkArgument(checkNotNull(input, "input") instanceof File, "this binder is only valid for File!");
checkNotNull(request, "request");
File dockerFile = (File) input;
File tmpDir = Files.createTempDir();
final File targetFile = new File(tmpDir + File.separator + "Dockerfile");
try {
Files.copy(dockerFile, targetFile);
File archive = Archives.tar(tmpDir, File.createTempFile("archive", ".tar"));
FileInputStream data = new FileInputStream(archive);
Payload payload = Payloads.newInputStreamPayload(data);
payload.getContentMetadata().setContentLength(data.getChannel().size());
payload.getContentMetadata().setContentType("application/tar");
request.setPayload(payload);
} catch (IOException e) {
logger.error(e, "Couldn't create a tarball for %s", targetFile);
throw Throwables.propagate(e);
}
return request;
}
}

16
apis/docker/src/main/java/org/jclouds/docker/features/RemoteApi.java
View File

@ -17,7 +17,6 @@
package org.jclouds.docker.features;
import java.io.Closeable;
import java.io.File;
import java.io.InputStream;
import java.util.Set;
@ -32,7 +31,6 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import org.jclouds.Fallbacks;
import org.jclouds.docker.binders.BindInputStreamToRequest;
import org.jclouds.docker.domain.Config;
import org.jclouds.docker.domain.Container;
import org.jclouds.docker.domain.HostConfig;
@ -255,18 +253,4 @@ public interface RemoteApi extends Closeable {
@Path("/build")
@Headers(keys = "Content-Type", values = "application/tar")
InputStream build(Payload inputStream, BuildOptions options);
/**
* Build an image from Dockerfile via stdin
*
* @param dockerFile The file to be compressed with one of the following algorithms: identity, gzip, bzip2, xz.*
* @param options the image build's options (@see BuildOptions)
* @return a stream of the build execution
*/
@Named("image:build")
@POST
@Path("/build")
@Headers(keys = "Content-Type", values = "application/tar")
InputStream build(@BinderParam(BindInputStreamToRequest.class) File dockerFile, BuildOptions options);
}

60
apis/docker/src/main/java/org/jclouds/docker/features/internal/Archives.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.docker.features.internal;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Iterables.getLast;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import com.google.common.base.Splitter;
import com.google.common.io.Files;
public class Archives {
public static File tar(File baseDir, String archivePath) throws IOException {
return tar(baseDir, new File(archivePath));
}
public static File tar(File baseDir, File tarFile) throws IOException {
// Check that the directory is a directory, and get its contents
checkArgument(baseDir.isDirectory(), "%s is not a directory", baseDir);
File[] files = baseDir.listFiles();
String token = getLast(Splitter.on("/").split(baseDir.getAbsolutePath()));
TarArchiveOutputStream tos = new TarArchiveOutputStream(new FileOutputStream(tarFile));
tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
try {
for (File file : files) {
TarArchiveEntry tarEntry = new TarArchiveEntry(file);
tarEntry.setName("/" + getLast(Splitter.on(token).split(file.toString())));
tos.putArchiveEntry(tarEntry);
if (!file.isDirectory()) {
Files.asByteSource(file).copyTo(tos);
}
tos.closeArchiveEntry();
}
} finally {
tos.close();
}
return tarFile;
}
}

67
apis/docker/src/test/java/org/jclouds/docker/binders/BindInputStreamToRequestTest.java
View File

@ -1,67 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.docker.binders;
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import org.jclouds.http.HttpRequest;
import org.testng.annotations.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import static org.testng.Assert.assertEquals;
import static org.testng.AssertJUnit.assertTrue;
@Test(groups = "unit", testName = "BindInputStreamToRequestTest")
public class BindInputStreamToRequestTest {
@Test
public void testBindInputStreamToRequest() throws IOException {
BindInputStreamToRequest binder = new BindInputStreamToRequest();
HttpRequest request = HttpRequest.builder().method("GET").endpoint("http://test").build();
request = binder.bindToRequest(request, File.createTempFile("dockerfile", ""));
String rawContent = CharStreams.toString(new InputStreamReader((FileInputStream) request.getPayload().getRawContent(), Charsets.UTF_8));
assertTrue(rawContent.startsWith("Dockerfile"));
assertEquals(request.getPayload().getContentMetadata().getContentType(), "application/tar");
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testBindInputStreamToRequestWithObjectAsInput() throws IOException {
BindInputStreamToRequest binder = new BindInputStreamToRequest();
HttpRequest request = HttpRequest.builder().method("GET").endpoint("http://test").build();
request = binder.bindToRequest(request, new Object());
String rawContent = CharStreams.toString(new InputStreamReader((FileInputStream) request.getPayload().getRawContent(), Charsets.UTF_8));
assertTrue(rawContent.startsWith("Dockerfile"));
assertEquals(request.getPayload().getContentMetadata().getContentType(), "application/tar");
}
@Test(expectedExceptions = NullPointerException.class)
public void testBindInputStreamToRequestWithNullInput() throws IOException {
BindInputStreamToRequest binder = new BindInputStreamToRequest();
HttpRequest request = HttpRequest.builder().method("GET").endpoint("http://test").build();
request = binder.bindToRequest(request, null);
String rawContent = CharStreams.toString(new InputStreamReader((FileInputStream) request.getPayload().getRawContent(), Charsets.UTF_8));
assertTrue(rawContent.startsWith("Dockerfile"));
assertEquals(request.getPayload().getContentMetadata().getContentType(), "application/tar");
}
}

52
apis/docker/src/test/java/org/jclouds/docker/compute/BaseDockerApiLiveTest.java
View File

@ -16,30 +16,30 @@
*/
package org.jclouds.docker.compute;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.CharStreams;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import com.google.inject.Module;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Properties;
import org.jboss.shrinkwrap.api.GenericArchive;
import org.jboss.shrinkwrap.api.ShrinkWrap;
import org.jboss.shrinkwrap.api.asset.ClassLoaderAsset;
import org.jboss.shrinkwrap.api.exporter.TarExporter;
import org.jclouds.Constants;
import org.jclouds.apis.BaseApiLiveTest;
import org.jclouds.docker.DockerApi;
import org.jclouds.docker.features.internal.Archives;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.sshj.config.SshjSshClientModule;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.Properties;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.CharStreams;
import com.google.common.io.Closeables;
import com.google.inject.Module;
@Test(groups = "live")
public class BaseDockerApiLiveTest extends BaseApiLiveTest<DockerApi> {
@ -72,20 +72,12 @@ public class BaseDockerApiLiveTest extends BaseApiLiveTest<DockerApi> {
return result;
}
protected Payload createPayload() throws IOException {
String folderPath = System.getProperty("user.dir") + "/docker/src/test/resources";
File parentDir = new File(folderPath + "/archive");
parentDir.mkdirs();
URL url = Resources.getResource("Dockerfile");
String content = Resources.toString(url, Charsets.UTF_8);
final File dockerfile = new File(parentDir.getAbsolutePath() + File.separator + "Dockerfile");
Files.write(content.getBytes(), dockerfile);
File archive = Archives.tar(parentDir, folderPath + "/archive/archive.tar");
FileInputStream data = new FileInputStream(archive);
Payload payload = Payloads.newInputStreamPayload(data);
payload.getContentMetadata().setContentLength(data.getChannel().size());
payload.getContentMetadata().setContentType("application/tar");
return payload;
}
public static Payload tarredDockerfile() throws IOException {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
ShrinkWrap.create(GenericArchive.class, "archive.tar")
.add(new ClassLoaderAsset("Dockerfile"), "Dockerfile")
.as(TarExporter.class).exportTo(bytes);
return Payloads.newByteArrayPayload(bytes.toByteArray());
}
}

30
apis/docker/src/test/java/org/jclouds/docker/features/RemoteApiLiveTest.java
View File

@ -16,10 +16,16 @@
*/
package org.jclouds.docker.features;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.io.Resources;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import org.jclouds.docker.compute.BaseDockerApiLiveTest;
import org.jclouds.docker.domain.Config;
import org.jclouds.docker.domain.Container;
@ -30,16 +36,9 @@ import org.jclouds.docker.options.DeleteImageOptions;
import org.jclouds.rest.ResourceNotFoundException;
import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
@Test(groups = "live", testName = "RemoteApiLiveTest", singleThreaded = true)
public class RemoteApiLiveTest extends BaseDockerApiLiveTest {
@ -104,7 +103,7 @@ public class RemoteApiLiveTest extends BaseDockerApiLiveTest {
public void testBuildImage() throws IOException, InterruptedException, URISyntaxException {
BuildOptions options = BuildOptions.Builder.tag("testBuildImage").verbose(false).nocache(false);
InputStream buildImageStream = api().build(new File(Resources.getResource("Dockerfile").toURI()), options);
InputStream buildImageStream = api().build(tarredDockerfile(), options);
String buildStream = consumeStream(buildImageStream, false);
Iterable<String> splitted = Splitter.on("\n").split(buildStream.replace("\r", "").trim());
String lastStreamedLine = Iterables.getLast(splitted).trim();
@ -117,5 +116,4 @@ public class RemoteApiLiveTest extends BaseDockerApiLiveTest {
private RemoteApi api() {
return api.getRemoteApi();
}
}

39
apis/docker/src/test/java/org/jclouds/docker/features/RemoteApiMockTest.java
View File

@ -315,22 +315,6 @@ public class RemoteApiMockTest extends BaseDockerMockTest {
}
}
public void testBuildContainer() throws Exception {
MockWebServer server = mockWebServer();
server.enqueue(new MockResponse().setResponseCode(200));
DockerApi api = api(server.getUrl("/"));
RemoteApi remoteApi = api.getRemoteApi();
File dockerFile = File.createTempFile("docker", "tmp");
try {
remoteApi.build(dockerFile, BuildOptions.NONE);
assertRequestHasCommonFields(server.takeRequest(), "POST", "/build");
} finally {
dockerFile.delete();
api.close();
server.shutdown();
}
}
public void testBuildContainerUsingPayload() throws Exception {
MockWebServer server = mockWebServer();
server.enqueue(new MockResponse().setResponseCode(200));
@ -350,27 +334,4 @@ public class RemoteApiMockTest extends BaseDockerMockTest {
server.shutdown();
}
}
public void testBuildNonexistentContainer() throws Exception {
MockWebServer server = mockWebServer();
server.enqueue(new MockResponse().setResponseCode(404));
DockerApi api = api(server.getUrl("/"));
RemoteApi remoteApi = api.getRemoteApi();
File dockerFile = File.createTempFile("docker", "tmp");
try {
try {
remoteApi.build(dockerFile, BuildOptions.NONE);
fail("Build container must fail on 404");
} catch (ResourceNotFoundException ex) {
// Expected exception
}
} finally {
dockerFile.delete();
api.close();
server.shutdown();
}
}
}

112
apis/docker/src/test/java/org/jclouds/docker/features/internal/ArchivesTest.java
View File

@ -1,112 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.docker.features.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.getOnlyElement;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import javax.annotation.Resource;
import javax.inject.Named;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarUtils;
import org.jclouds.compute.reference.ComputeServiceConstants;
import org.jclouds.logging.Logger;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import com.beust.jcommander.internal.Lists;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
@Test(groups = "unit", testName = "ArchivesTest")
public class ArchivesTest {
@Resource
@Named(ComputeServiceConstants.COMPUTE_LOGGER)
protected Logger logger = Logger.NULL;
private File tmpDir;
private File outputDir;
private long checkSum;
@BeforeClass
private void init() throws IOException {
tmpDir = Files.createTempDir();
outputDir = Files.createTempDir();
File sampleFile = writeSampleFile("test", "this is a test to tar a hierarchy of folders and files\n");
checkSum = TarUtils.computeCheckSum(Files.asByteSource(sampleFile).read());
}
public void testTarSingleFile() throws Exception {
File archive = Archives.tar(tmpDir, new File(outputDir + File.separator + "test.tar.gz"));
List<File> untarredFiles = unTar(archive, outputDir);
File untarredSampleFile = getOnlyElement(untarredFiles, null);
assertNotNull(untarredSampleFile);
assertTrue(checkSum == TarUtils.computeCheckSum(Files.asByteSource(untarredSampleFile).read()));
}
private List<File> unTar(final File inputFile, final File outputDir) throws Exception {
final List<File> untarredFiles = Lists.newArrayList();
final InputStream is = new FileInputStream(inputFile);
final TarArchiveInputStream tarArchiveInputStream = (TarArchiveInputStream)
new ArchiveStreamFactory().createArchiveInputStream("tar", is);
TarArchiveEntry entry;
while ((entry = (TarArchiveEntry) tarArchiveInputStream.getNextEntry()) != null) {
final File outputFile = new File(outputDir, entry.getName());
if (entry.isDirectory()) {
if (!outputFile.exists()) {
if (!outputFile.mkdirs()) {
throw new IllegalStateException(String.format("Couldn't create directory %s.", outputFile.getAbsolutePath()));
}
}
} else {
OutputStream outputFileStream = new FileOutputStream(outputFile);
ByteStreams.copy(tarArchiveInputStream, outputFileStream);
outputFileStream.close();
}
untarredFiles.add(outputFile);
}
tarArchiveInputStream.close();
return untarredFiles;
}
private File writeSampleFile(String fileName, final String contents) {
checkNotNull(fileName, "Provided file name for writing must NOT be null.");
checkNotNull(contents, "Unable to write null contents.");
File sampleFile = new File(tmpDir + File.separator + fileName);
try {
Files.write(contents.getBytes(), sampleFile);
} catch (IOException e) {
logger.error("ERROR trying to write to file '" + fileName + "' - " + e.toString());
Assert.fail();
}
return sampleFile;
}
}