diff --git a/packer/rpc/command_test.go b/packer/rpc/command_test.go
index 00276ef6e..238e163d6 100644
--- a/packer/rpc/command_test.go
+++ b/packer/rpc/command_test.go
@@ -1,7 +1,6 @@
 package rpc
 
 import (
-	"bytes"
 	"cgl.tideland.biz/asserts"
 	"github.com/mitchellh/packer/packer"
 	"net"
@@ -26,21 +25,6 @@ func (tc *TestCommand) Synopsis() string {
 	return "foo"
 }
 
-func testEnvironment() packer.Environment {
-	config := &packer.EnvironmentConfig{}
-	config.Ui = &packer.ReaderWriterUi{
-		new(bytes.Buffer),
-		new(bytes.Buffer),
-	}
-
-	env, err := packer.NewEnvironment(config)
-	if err != nil {
-		panic(err)
-	}
-
-	return env
-}
-
 // This starts a RPC server for the given command listening on the
 // given address. The RPC server is ready when "readyChan" receives a message
 // and the RPC server will quit when "stopChan" receives a message.
@@ -102,7 +86,7 @@ func TestRPCCommand(t *testing.T) {
 
 	clientComm := &ClientCommand{client}
 	runArgs := []string{"foo", "bar"}
-	testEnv := testEnvironment()
+	testEnv := &testEnvironment{}
 	exitCode := clientComm.Run(testEnv, runArgs)
 	synopsis := clientComm.Synopsis()
 
diff --git a/packer/rpc/environment.go b/packer/rpc/environment.go
index a26b8d7a2..bc91162d5 100644
--- a/packer/rpc/environment.go
+++ b/packer/rpc/environment.go
@@ -5,9 +5,9 @@ import (
 	"net/rpc"
 )
 
-// A EnvironmentClient is an implementation of the packer.Environment interface
+// A Environment is an implementation of the packer.Environment interface
 // where the actual environment is executed over an RPC connection.
-type EnvironmentClient struct {
+type Environment struct {
 	client *rpc.Client
 }
 
@@ -16,3 +16,60 @@ type EnvironmentClient struct {
 type EnvironmentServer struct {
 	env packer.Environment
 }
+
+type EnvironmentCliArgs struct {
+	Args []string
+}
+
+func (e *Environment) BuilderFactory() packer.BuilderFactory {
+	var reply string
+	e.client.Call("Environment.BuilderFactory", new(interface{}), &reply)
+
+	// TODO: error handling
+	client, _ := rpc.Dial("tcp", reply)
+	return &BuilderFactory{client}
+}
+
+func (e *Environment) Cli(args []string) (result int) {
+	rpcArgs := &EnvironmentCliArgs{args}
+	e.client.Call("Environment.Cli", rpcArgs, &result)
+	return
+}
+
+func (e *Environment) Ui() packer.Ui {
+	var reply string
+	e.client.Call("Environment.Ui", new(interface{}), &reply)
+
+	// TODO: error handling
+	client, _ := rpc.Dial("tcp", reply)
+	return &Ui{client}
+}
+
+func (e *EnvironmentServer) BuilderFactory(args *interface{}, reply *string) error {
+	bf := e.env.BuilderFactory()
+
+	// Wrap that up into a server, and server a single connection back
+	server := NewServer()
+	server.RegisterBuilderFactory(bf)
+	server.StartSingle()
+
+	*reply = server.Address()
+	return nil
+}
+
+func (e *EnvironmentServer) Cli(args *EnvironmentCliArgs, reply *int) error {
+	*reply = e.env.Cli(args.Args)
+	return nil
+}
+
+func (e *EnvironmentServer) Ui(args *interface{}, reply *string) error {
+	ui := e.env.Ui()
+
+	// Wrap it
+	server := NewServer()
+	server.RegisterUi(ui)
+	server.StartSingle()
+
+	*reply = server.Address()
+	return nil
+}
diff --git a/packer/rpc/environment_test.go b/packer/rpc/environment_test.go
new file mode 100644
index 000000000..cb1ca399a
--- /dev/null
+++ b/packer/rpc/environment_test.go
@@ -0,0 +1,85 @@
+package rpc
+
+import (
+	"cgl.tideland.biz/asserts"
+	"github.com/mitchellh/packer/packer"
+	"net/rpc"
+	"testing"
+)
+
+var testEnvBF = &testBuilderFactory{}
+var testEnvUi = &testUi{}
+
+type testEnvironment struct {
+	bfCalled bool
+	cliCalled bool
+	cliArgs []string
+	uiCalled bool
+}
+
+func (e *testEnvironment) BuilderFactory() packer.BuilderFactory {
+	e.bfCalled = true
+	return testEnvBF
+}
+
+func (e *testEnvironment) Cli(args []string) int {
+	e.cliCalled = true
+	e.cliArgs = args
+	return 42
+}
+
+func (e *testEnvironment) Ui() packer.Ui {
+	e.uiCalled = true
+	return testEnvUi
+}
+
+func TestEnvironmentRPC(t *testing.T) {
+	assert := asserts.NewTestingAsserts(t, true)
+
+	// Create the interface to test
+	e := &testEnvironment{}
+
+	// Start the server
+	server := NewServer()
+	server.RegisterEnvironment(e)
+	server.Start()
+	defer server.Stop()
+
+	// Create the client over RPC and run some methods to verify it works
+	client, err := rpc.Dial("tcp", server.Address())
+	assert.Nil(err, "should be able to connect")
+
+	// Test BuilderFactory
+	eClient := &Environment{client}
+	bf := eClient.BuilderFactory()
+	assert.True(e.bfCalled, "BuilderFactory should be called")
+
+	// Test calls on the factory
+	_ = bf.CreateBuilder("foo")
+	assert.True(testEnvBF.createCalled, "create should be called on BF")
+
+	// Test Cli
+	cliArgs := []string{"foo", "bar"}
+	result := eClient.Cli(cliArgs)
+	assert.True(e.cliCalled, "CLI should be called")
+	assert.Equal(e.cliArgs, cliArgs, "args should match")
+	assert.Equal(result, 42, "result shuld be 42")
+
+	// Test Ui
+	ui := eClient.Ui()
+	assert.True(e.uiCalled, "Ui should've been called")
+
+	// Test calls on the Ui
+	ui.Say("format")
+	assert.True(testEnvUi.sayCalled, "Say should be called")
+	assert.Equal(testEnvUi.sayFormat, "format", "format should match")
+}
+
+func TestEnvironment_ImplementsEnvironment(t *testing.T) {
+	assert := asserts.NewTestingAsserts(t, true)
+
+	var realVar packer.Environment
+	e := &Environment{nil}
+
+	assert.Implementor(e, &realVar, "should be an Environment")
+}
diff --git a/packer/rpc/server.go b/packer/rpc/server.go
index b8b2e9ee1..0932872ce 100644
--- a/packer/rpc/server.go
+++ b/packer/rpc/server.go
@@ -41,6 +41,10 @@ func (s *Server) RegisterBuilderFactory(b packer.BuilderFactory) {
 	s.server.RegisterName("BuilderFactory", &BuilderFactoryServer{b})
 }
 
+func (s *Server) RegisterEnvironment(e packer.Environment) {
+	s.server.RegisterName("Environment", &EnvironmentServer{e})
+}
+
 func (s *Server) RegisterUi(ui packer.Ui) {
 	s.server.RegisterName("Ui", &UiServer{ui})
 }