OPENJPA-2165 Added support for non-db-ordered list proxies that provide the ability to do non-indexed add or remove operations without loading the collection from the database. Testcases and documentation will follow in future commits.

git-svn-id: https://svn.apache.org/repos/asf/openjpa/trunk@1306449 13f79535-47bb-0310-9956-ffa450edef68

openjpa-jdbc/src/main/java/org/apache/openjpa/jdbc/kernel/JDBCStoreManager.java

@@ -242,7 +242,7 @@ public class JDBCStoreManager implements StoreManager, JDBCStore {
     }
     
     protected DataSource getDataSource() {
-    	return _ds;
+        return _ds;
     }
 
     public boolean exists(OpenJPAStateManager sm, Object context) {
@@ -641,31 +641,37 @@ public class JDBCStoreManager implements StoreManager, JDBCStore {
                 && mapping.customLoad(sm, this, null, jfetch))
                 removeLoadedFields(sm, fields);
 
+            
             //### select is kind of a big object, and in some cases we don't
             //### use it... would it be worth it to have a small shell select
             //### object that only creates a real select when actually used?
+            //### Delayed proxy specific optimization: If the only fields that 
+            //### need to be loaded are delayed proxies, building the select is 
+            //### not necessary.
 
-            Select sel = _sql.newSelect();
+            if (!isDelayedLoadOnly(sm, fields, mapping)) {
-            if (select(sel, mapping, Select.SUBS_EXACT, sm, fields, jfetch,
+	            Select sel = _sql.newSelect();
-                EagerFetchModes.EAGER_JOIN, true, false)) {
+	            if (select(sel, mapping, Select.SUBS_EXACT, sm, fields, jfetch,
-                sel.wherePrimaryKey(sm.getObjectId(), mapping, this);
+	                EagerFetchModes.EAGER_JOIN, true, false)) {
-                if (_log.isTraceEnabled()) {
+	                sel.wherePrimaryKey(sm.getObjectId(), mapping, this);
-                    _log.trace("load: "+mapping.getDescribedType()+" oid: "+sm.getObjectId()); 
+	                if (_log.isTraceEnabled()) {
-                }
+	                    _log.trace("load: "+mapping.getDescribedType()+" oid: "+sm.getObjectId()); 
-                res = sel.execute(this, jfetch, lockLevel);
+	                }
-                try {
+	                res = sel.execute(this, jfetch, lockLevel);
-                 	if (isEmptyResult(res))
+	                try {
-                        return false;
+	                    if (isEmptyResult(res))
-                    load(mapping, sm, jfetch, res);
+	                        return false;
-                } finally {
+	                    load(mapping, sm, jfetch, res);
-                    res.close();
+	                } finally {
-                }
+	                    res.close();
+	                }
+	            }
             }
 
             // now allow the fields to load themselves individually too
             FieldMapping[] fms = mapping.getFieldMappings();
             for (int i = 0; i < fms.length; i++)
-                if (fields.get(i) && !sm.getLoaded().get(i)) {
+                if (fields.get(i) && (!sm.getLoaded().get(i) || sm.isDelayed(i))) {
                     if (_log.isTraceEnabled()) {
                         _log.trace("load field: '"+ fms[i].getName() + "' for oid="+sm.getObjectId()
                             +" "+mapping.getDescribedType());
@@ -681,6 +687,30 @@ public class JDBCStoreManager implements StoreManager, JDBCStore {
         }
     }
 
+    private boolean isDelayedLoadOnly(OpenJPAStateManager sm, BitSet fields, ClassMapping mapping) {
+        if (!sm.getContext().getConfiguration().getProxyManagerInstance().getDelayCollectionLoading()) {
+            return false;
+        }
+        boolean allDelayed = false;
+        if (!fields.isEmpty()) {
+            FieldMapping[] fms = mapping.getFieldMappings();
+            int fCount = 0;
+            int dfCount = 0;
+            for (int i = fields.nextSetBit(0); i < fms.length; i++) {
+                if (fields.get(i)) {
+                    fCount++;
+                    if (!(fms[i].isDelayCapable() && (!sm.getLoaded().get(i) || sm.isDelayed(i)))) {
+                        break;
+                    } else {
+                        dfCount++;
+                    }
+                }
+            }
+            allDelayed = (fCount == dfCount);
+        }
+        return allDelayed;
+    }
+    
     /**
      * Return a list formed by removing all loaded fields from the given one.
      */
@@ -1080,10 +1110,10 @@ public class JDBCStoreManager implements StoreManager, JDBCStore {
                         eagerToMany = fms[i];
                     else
                         fms[i].loadEagerJoin(sm, this, 
-                        	fetch.traverseJDBC(fms[i]), res);
+                            fetch.traverseJDBC(fms[i]), res);
                 } else if (eres != null) {
                     processed = fms[i].loadEagerParallel(sm, this, 
-                    	fetch.traverseJDBC(fms[i]), eres);
+                        fetch.traverseJDBC(fms[i]), eres);
                     if (processed != eres)
                         res.putEager(fms[i], processed);
                 } else {
@@ -1332,18 +1362,18 @@ public class JDBCStoreManager implements StoreManager, JDBCStore {
             if (esel != null) {
                 if (esel == sel)
                     fms[i].selectEagerJoin(sel, sm, this, 
-                    	fetch.traverseJDBC(fms[i]), eager);
+                        fetch.traverseJDBC(fms[i]), eager);
                 else
                     fms[i].selectEagerParallel(esel, sm, this, 
-                    	fetch.traverseJDBC(fms[i]), eager);
+                        fetch.traverseJDBC(fms[i]), eager);
                 seld = Math.max(0, seld);
             } else if (requiresSelect(fms[i], sm, fields, fetch)) {
                 fseld = fms[i].select(sel, sm, this, 
-                	fetch.traverseJDBC(fms[i]), eager);
+                    fetch.traverseJDBC(fms[i]), eager);
                 seld = Math.max(fseld, seld);
             } else if (optSelect(fms[i], sel, sm, fetch)) {
                 fseld = fms[i].select(sel, sm, this, 
-                	fetch.traverseJDBC(fms[i]), EagerFetchModes.EAGER_NONE);
+                    fetch.traverseJDBC(fms[i]), EagerFetchModes.EAGER_NONE);
 
                 // don't upgrade seld to > 0 based on these fields, since
                 // they're not in the calculated field set
@@ -1425,12 +1455,12 @@ public class JDBCStoreManager implements StoreManager, JDBCStore {
             fms = subMappings[i].getDefinedFieldMappings();
             for (int j = 0; j < fms.length; j++) {
                 // make sure in one of configured fetch groups
-            	if (fetch.requiresFetch(fms[j]) != FetchConfiguration.FETCH_LOAD
+                if (fetch.requiresFetch(fms[j]) != FetchConfiguration.FETCH_LOAD
                     && ((!fms[j].isInDefaultFetchGroup() 
                     && fms[j].isDefaultFetchGroupExplicit())
                     || fms[j].supportsSelect(sel, Select.TYPE_TWO_PART, sm, this, 
                     fetch) <= 0)) 
-            		continue;
+                    continue;
 
                 // if we can join to the subclass, do so; much better chance
                 // that the field will be able to select itself without joins

openjpa-jdbc/src/main/java/org/apache/openjpa/jdbc/meta/FieldMapping.java

@@ -1359,4 +1359,9 @@ public class FieldMapping
     public boolean hasMapsIdCols() {
         return _hasMapsIdCols;
     }
+    
+    @Override
+    public boolean isDelayCapable() {
+        return (getOrderColumn() == null && super.isDelayCapable());
+    }
 }

openjpa-jdbc/src/main/java/org/apache/openjpa/jdbc/meta/strats/EmbedFieldStrategy.java

@@ -1252,6 +1252,21 @@ public class EmbedFieldStrategy
         public Object replaceObjectField(PersistenceCapable pc, int field) {
             throw new InternalException();
         }
+
+        @Override
+        public boolean isDelayed(int field) {
+            return false;
+        }
+        
+        @Override
+        public void setDelayed(int field, boolean delay) {
+            throw new InternalException();
+        }
+        
+        @Override
+        public void loadDelayedField(int field) {
+            throw new UnsupportedOperationException();
+        }
     }
 
     /**

openjpa-jdbc/src/main/java/org/apache/openjpa/jdbc/meta/strats/StoreCollectionFieldStrategy.java

@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.openjpa.enhance.FieldManager;
 import org.apache.openjpa.enhance.PersistenceCapable;
 import org.apache.openjpa.jdbc.kernel.JDBCFetchConfiguration;
 import org.apache.openjpa.jdbc.kernel.JDBCStore;
@@ -42,13 +43,13 @@ import org.apache.openjpa.jdbc.sql.SelectExecutor;
 import org.apache.openjpa.jdbc.sql.Union;
 import org.apache.openjpa.kernel.OpenJPAStateManager;
 import org.apache.openjpa.kernel.StateManagerImpl;
-import org.apache.openjpa.lib.util.Localizer;
 import org.apache.openjpa.meta.ClassMetaData;
 import org.apache.openjpa.meta.JavaTypes;
 import org.apache.openjpa.util.ChangeTracker;
 import org.apache.openjpa.util.Id;
 import org.apache.openjpa.util.OpenJPAId;
 import org.apache.openjpa.util.Proxy;
+import org.apache.openjpa.util.DelayedProxy;
 
 /**
  * Base class for strategies that are stored as a collection, even if
@@ -491,8 +492,20 @@ public abstract class StoreCollectionFieldStrategy
     public void load(final OpenJPAStateManager sm, final JDBCStore store,
         final JDBCFetchConfiguration fetch)
         throws SQLException {
+        
+        Object coll = null;
+        boolean delayed = sm.isDelayed(field.getIndex());
+        if (!delayed && field.isDelayCapable()) {
+            coll = sm.newProxy(field.getIndex());
+            if (coll instanceof DelayedProxy) {
+                sm.storeObject(field.getIndex(), coll);
+                sm.setDelayed(field.getIndex(), true);
+                return;
+            }
+        }
+        
         if (field.isLRS()) {
-            Proxy coll = newLRSProxy();
+            Proxy pcoll = newLRSProxy();
 
             // if this is ordered we need to know the next seq to use in case
             // objects are added to the collection
@@ -513,13 +526,13 @@ public abstract class StoreCollectionFieldStrategy
                 Result res = sel.execute(store, fetch);
                 try {
                     res.next();
-                    coll.getChangeTracker().setNextSequence
+                    pcoll.getChangeTracker().setNextSequence
                         (res.getInt(field) + 1);
                 } finally {
                     res.close();
                 }
             }
-            sm.storeObjectField(field.getIndex(), coll);
+            sm.storeObjectField(field.getIndex(), pcoll);
             return;
         }
 
@@ -537,14 +550,27 @@ public abstract class StoreCollectionFieldStrategy
         });
 
         // create proxy
-        Object coll;
         ChangeTracker ct = null;
-        if (field.getTypeCode() == JavaTypes.ARRAY)
+        if (delayed) {
-            coll = new ArrayList();
+            if (sm.isDetached() || sm.getOwner() == null) {
-        else {
+                sm.getPersistenceCapable().pcProvideField(field.getIndex());
-            coll = sm.newProxy(field.getIndex());
+                coll = 
+                    ((FieldManager)sm.getPersistenceCapable().pcGetStateManager()).fetchObjectField(field.getIndex());
+            } else {
+                coll = sm.fetchObjectField(field.getIndex());
+            }
             if (coll instanceof Proxy)
                 ct = ((Proxy) coll).getChangeTracker();
+        } else {
+            if (field.getTypeCode() == JavaTypes.ARRAY)
+                coll = new ArrayList();
+            else {
+                if (coll == null) {
+                    coll = sm.newProxy(field.getIndex());
+                }
+                if (coll instanceof Proxy)
+                    ct = ((Proxy) coll).getChangeTracker();
+            }
         }
 
         // load values
@@ -564,12 +590,14 @@ public abstract class StoreCollectionFieldStrategy
             res.close();
         }
 
-        // set into sm
+        // if not a delayed collection, set into sm
-        if (field.getTypeCode() == JavaTypes.ARRAY)
+        if (!delayed) {
-            sm.storeObject(field.getIndex(), JavaTypes.toArray
+            if (field.getTypeCode() == JavaTypes.ARRAY)
-                ((Collection) coll, field.getElement().getType()));
+                sm.storeObject(field.getIndex(), JavaTypes.toArray
-        else
+                    ((Collection) coll, field.getElement().getType()));
-            sm.storeObject(field.getIndex(), coll);
+            else
+                sm.storeObject(field.getIndex(), coll);
+        }
     }
 
     /**

openjpa-jdbc/src/test/java/org/apache/openjpa/jdbc/kernel/TestUpdateManagerFlushException.java

@@ -690,6 +690,18 @@ public class TestUpdateManagerFlushException extends /* Abstract */TestCase {
         public String fetchStringField(int fieldIndex) {
             return null;
         }
+
+        @Override
+        public boolean isDelayed(int field) {
+            return false;
+        }
+
+        @Override
+        public void setDelayed(int field, boolean delay) {
+        }
+
+        public void loadDelayedField(int field) {
+        }
     }
 
     /*

openjpa-kernel/src/main/java/org/apache/openjpa/kernel/DetachedStateManager.java

@@ -995,4 +995,19 @@ public class DetachedStateManager
         if (_lock != null)
             _lock.unlock();
     }
+    
+    @Override
+    public boolean isDelayed(int field) {
+        return false;
+    }
+    
+    @Override
+    public void setDelayed(int field, boolean delay) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void loadDelayedField(int field) {
+        throw new UnsupportedOperationException();
+    }
 }

openjpa-kernel/src/main/java/org/apache/openjpa/kernel/DetachedValueStateManager.java

@@ -595,5 +595,20 @@ public class DetachedValueStateManager
     public Object replaceObjectField(PersistenceCapable pc, int idx) {
         throw new UnsupportedOperationException();
     }
+    
+    @Override
+    public boolean isDelayed(int field) {
+        return false;
+    }
+    
+    @Override
+    public void setDelayed(int field, boolean delay) {
+        throw new UnsupportedOperationException();
+    }
+    
+    @Override
+    public void loadDelayedField(int field) {
+        throw new UnsupportedOperationException();
+    }
 }
 

openjpa-kernel/src/main/java/org/apache/openjpa/kernel/ObjectIdStateManager.java

@@ -739,4 +739,19 @@ public class ObjectIdStateManager
             Reflection.set(_oid, Reflection.findSetter(_oid.getClass(),
                 fmd.getName(), fmd.getDeclaredType(), true), val);
 	}
+    
+    @Override
+    public boolean isDelayed(int field) {
+        return false;
+    }
+
+    @Override
+    public void setDelayed(int field, boolean delay) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void loadDelayedField(int field) {
+        throw new UnsupportedOperationException();
+    }
 }

openjpa-kernel/src/main/java/org/apache/openjpa/kernel/OpenJPAStateManager.java

@@ -498,5 +498,37 @@ public interface OpenJPAStateManager
      * @since 0.3.1
      */
     public void setRemote (int field, Object value);
+    
+    /**
+     * Some field types (collection proxies) support delayed loading.  Delayed loading
+     * is a step beyond lazy loading.  Delayed load allows an instance of a field to be 
+     * returned without actually loading it.
+     * 
+     * @param field
+     * @return true if the field is setup for delayed access
+     */
+    public boolean isDelayed(int field);
+    
+    /**
+     * Some field types (collection proxies) support delayed loading.  Delayed loading
+     * is a step beyond lazy loading.  Delayed load allows an instance of a field to be 
+     * returned without actually loading it.
+     * 
+     * @param field
+     */
+    public void setDelayed(int field, boolean delay);
+    
+    /**
+     * If a field was marked delayed in a previous load operation this method can be
+     * used to load the field.
+     * @param field
+     */
+    public void loadDelayedField(int field);
+    
+    /**
+     * Fetch an object field by index.
+     * @param field
+     */
+    public Object fetchObjectField(int field);
 }
 

openjpa-kernel/src/main/java/org/apache/openjpa/kernel/StateManagerImpl.java

@@ -118,6 +118,7 @@ public class StateManagerImpl
     protected BitSet _loaded = null;
     private BitSet _dirty = null;
     private BitSet _flush = null;
+    private BitSet _delayed = null;
     private int _flags = 0;
 
     // id is the state manager identity; oid is the persistent identity.  oid
@@ -1600,6 +1601,58 @@ public class StateManagerImpl
         }
     }
 
+    public boolean isDelayed(int field) {
+        if (_delayed == null) {
+            return false;
+        }
+        return _delayed.get(field);
+    }
+
+    public void setDelayed(int field, boolean delay) {
+        if (_delayed == null) {
+            _delayed = new BitSet();
+        }
+        if (delay) {
+            _delayed.set(field);
+        } else {
+            _delayed.clear(field);
+        }
+    }
+
+    /**
+     * Loads a delayed access field.
+     * @param field
+     */
+    public void loadDelayedField(int field) {
+        if (!isDelayed(field)) {
+            return;
+        }
+
+        try {
+            beforeRead(field);
+        } catch (RuntimeException re) {
+            throw translate(re);
+        }
+        lock();
+        try {
+            boolean active = _broker.isActive();
+            int lockLevel = calculateLockLevel(active, false, null);
+            BitSet fields = new BitSet();
+            fields.set(field);
+            if (!_broker.getStoreManager().load(this, fields, _broker.getFetchConfiguration(), lockLevel, null)) {
+                throw new ObjectNotFoundException(_loc.get("del-instance", _meta.getDescribedType(), _oid)).
+                    setFailedObject(getManagedInstance());
+            }
+            // Cleared the delayed bit
+            _delayed.clear(field);
+            obtainLocks(active, false, lockLevel, null, null);
+        } catch (RuntimeException re) {
+            throw translate(re);
+        } finally {
+            unlock();
+        }
+    }
+
     /**
      * Load the given field before access.
      */
@@ -3420,4 +3473,8 @@ public class StateManagerImpl
     public void setPc(PersistenceCapable pc) {
         _pc = pc;
     }
+
+    public void setBroker(BrokerImpl ctx) {
+        _broker = ctx;
+    }
 }

openjpa-kernel/src/main/java/org/apache/openjpa/meta/FieldMetaData.java

@@ -61,6 +61,7 @@ import org.apache.openjpa.util.Exceptions;
 import org.apache.openjpa.util.InternalException;
 import org.apache.openjpa.util.MetaDataException;
 import org.apache.openjpa.util.OpenJPAException;
+import org.apache.openjpa.util.ProxyManager;
 import org.apache.openjpa.util.UnsupportedException;
 import org.apache.openjpa.util.ImplHelper;
 import org.apache.openjpa.util.UserException;
@@ -222,6 +223,7 @@ public class FieldMetaData
 
     private boolean _persistentCollection = false; 
 
+    private Boolean _delayCapable = null;
     /**
      * Constructor.
      *
@@ -2405,4 +2407,27 @@ public class FieldMetaData
     	return _relationType;
     }
     private class Unknown{};
+    
+    public boolean isDelayCapable() {
+        if (_delayCapable != null) {
+            return _delayCapable.booleanValue();
+        }
+        if (getTypeCode() != JavaTypes.COLLECTION || isLRS()) {
+           _delayCapable = Boolean.FALSE;
+           return _delayCapable;
+        } else {
+            // Verify the proxy manager is configured to handle delay loading
+            ProxyManager pm = getRepository().getConfiguration().getProxyManagerInstance();
+            if (pm != null) {
+                _delayCapable = pm.getDelayCollectionLoading();
+            } else {
+                _delayCapable = Boolean.FALSE;
+            }
+        }
+        return _delayCapable;
+    }
+    
+    public void setDelayCapable(Boolean delayCapable) {
+        _delayCapable = delayCapable;
+    }
 }

openjpa-kernel/src/main/java/org/apache/openjpa/util/CollectionChangeTrackerImpl.java

@@ -31,9 +31,9 @@ public class CollectionChangeTrackerImpl
     extends AbstractChangeTracker
     implements CollectionChangeTracker {
 
-    private final Collection _coll;
+    protected final Collection _coll;
-    private final boolean _dups;
+    protected final boolean _dups;
-    private final boolean _order;
+    protected final boolean _order;
 
     /**
      * Constructor.

openjpa-kernel/src/main/java/org/apache/openjpa/util/DelayedArrayListProxy.java

@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.openjpa.util;
+
+import java.io.ObjectStreamException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import org.apache.openjpa.kernel.AutoDetach;
+import org.apache.openjpa.kernel.Broker;
+import org.apache.openjpa.kernel.BrokerFactory;
+import org.apache.openjpa.kernel.OpenJPAStateManager;
+
+public class DelayedArrayListProxy extends ArrayList
+  implements ProxyCollection, DelayedProxy
+{
+  private transient OpenJPAStateManager sm;
+  private transient OpenJPAStateManager _ownerSm;
+  private transient int field;
+  private transient CollectionChangeTracker changeTracker;
+  private transient Class<?> elementType;
+  private transient boolean _directAccess = false;
+  private transient BrokerFactory _brokerFactory = null;
+  private transient Broker _broker = null;
+  private transient OpenJPAStateManager _delayedSm;
+  private transient int _delayedField; 
+  private transient boolean dirtyCollection = true;
+  
+  public DelayedArrayListProxy()
+  {
+  }
+  
+  public DelayedArrayListProxy(Collection paramCollection)
+  {
+    super(paramCollection);
+  }
+
+  public DelayedArrayListProxy(int paramInt)
+  {
+    super(paramInt);
+  }
+
+  public void setOwner(OpenJPAStateManager paramOpenJPAStateManager, int paramInt)
+  {
+    // If clearing the owner of this proxy, store away what is necessary for 
+    // delayed loading 
+    if (paramOpenJPAStateManager == null && paramInt == -1 && sm != null) {
+        _delayedSm = sm;
+        _delayedField = field;
+    }
+    
+    this.sm = paramOpenJPAStateManager;
+    if (sm != null && sm.getPersistenceCapable() != null) {
+        _ownerSm = (OpenJPAStateManager) sm.getPersistenceCapable().pcGetStateManager();
+    }
+    this.field = paramInt;
+    if (sm != null && sm.getContext() != null) {
+        _brokerFactory = sm.getContext().getBroker().getBrokerFactory();
+    }
+  }
+  
+  public int getDelayedField() {
+      if (field == -1) {
+          return _delayedField;
+      }
+      return field;
+  }
+  
+  public OpenJPAStateManager getDelayedOwner() {
+      if (sm == null) {
+          return _delayedSm;
+      }
+      return sm;
+  }
+
+  public OpenJPAStateManager getOwner()
+  {
+    return sm;
+  }
+
+  public int getOwnerField()
+  {
+    return field;
+  }
+
+  public Object clone()
+  {
+    if (isDirectAccess()) {
+        return super.clone();
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    Proxy localProxy = (Proxy)super.clone();
+    localProxy.setOwner(null, 0);
+    return localProxy;
+  }
+
+  public ChangeTracker getChangeTracker()
+  {
+    return this.changeTracker;
+  }
+  
+  protected void setChangeTracker(CollectionChangeTracker ct) {
+      changeTracker = ct;
+  }
+
+  public Object copy(Object paramObject)
+  {
+    return new ArrayList((Collection)paramObject);
+  }
+
+  public Class getElementType()
+  {
+    return this.elementType;
+  }
+  
+  protected void setElementType(Class<?> elemType) {
+      elementType = elemType;
+  }
+
+  @Override
+  public ProxyCollection newInstance(Class paramClass, Comparator paramComparator, boolean paramBoolean1, 
+          boolean paramBoolean2)
+  {
+    DelayedArrayListProxy proxy = new DelayedArrayListProxy();
+    proxy.elementType = paramClass;
+    proxy.changeTracker = new DelayedCollectionChangeTrackerImpl(proxy, true, true, paramBoolean2);
+    return proxy;
+  }
+
+  public boolean add(Object paramObject)
+  {
+    if (_directAccess) {
+        return super.add(paramObject);
+    }
+    ProxyCollections.beforeAdd(this, paramObject);
+    boolean bool = super.add(paramObject);
+    return ProxyCollections.afterAdd(this, paramObject, bool);
+  }
+
+  public void add(int paramInt, Object paramObject)
+  {
+    if (!_directAccess) {
+        if (isDelayLoad()) {
+            load();
+        }
+    }
+    ProxyCollections.beforeAdd(this, paramInt, paramObject);
+    super.add(paramInt, paramObject);
+  }
+
+  public void clear()
+  {
+    if (!_directAccess) {
+        if (isDelayLoad()) {
+            load();
+        }
+        ProxyCollections.beforeClear(this); 
+    }
+    super.clear();
+  }
+
+  public boolean addAll(int paramInt, Collection paramCollection)
+  {
+    if (isDelayLoad()) {
+        load();
+    }
+    return ProxyCollections.addAll(this, paramInt, paramCollection);
+  }
+
+  public boolean addAll(Collection paramCollection)
+  {
+    if (_directAccess) {
+        return super.addAll(paramCollection);
+    }
+    return ProxyCollections.addAll(this, paramCollection);
+  }
+
+  public boolean remove(Object paramObject)
+  {
+      if (_directAccess) {
+          return super.remove(paramObject); 
+      }
+      ProxyCollections.beforeRemove(this, paramObject);
+      setDirectAccess(true);
+      boolean bool = super.remove(paramObject);
+      setDirectAccess(false);
+      return ProxyCollections.afterRemove(this, paramObject, bool);
+  }
+
+  public Object remove(int paramInt)
+  {
+    if (_directAccess) {
+        return super.remove(paramInt);
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    ProxyCollections.beforeRemove(this, paramInt);
+    Object localObject = super.remove(paramInt);
+    return ProxyCollections.afterRemove(this, paramInt, localObject);
+  }
+
+  public Object set(int paramInt, Object paramObject)
+  {
+    if (_directAccess) {
+        return super.set(paramInt, paramObject);
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    ProxyCollections.beforeSet(this, paramInt, paramObject);
+    Object localObject = super.set(paramInt, paramObject);
+    return ProxyCollections.afterSet(this, paramInt, paramObject, localObject);
+  }
+
+  public Iterator iterator()
+  {
+    if (_directAccess) {
+        return super.iterator();
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    Iterator localIterator = super.iterator();
+    return ProxyCollections.afterIterator(this, localIterator);
+  }
+
+  public ListIterator listIterator(int paramInt)
+  {
+    if (_directAccess) {
+        return super.listIterator(paramInt);
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    ListIterator localListIterator = super.listIterator(paramInt);
+    return ProxyCollections.afterListIterator(this, paramInt, localListIterator);
+  }
+
+  public ListIterator listIterator()
+  {
+    if (_directAccess) {
+        return super.listIterator();
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    ListIterator localListIterator = super.listIterator();
+    return ProxyCollections.afterListIterator(this, localListIterator);
+  }
+
+  public boolean removeAll(Collection paramCollection)
+  {
+    if (_directAccess) {
+        return super.removeAll(paramCollection);
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    return ProxyCollections.removeAll(this, paramCollection);
+  }
+
+  public boolean retainAll(Collection paramCollection)
+  {
+    if (_directAccess) {
+        return super.retainAll(paramCollection);
+    }
+    if (isDelayLoad()) {
+        load();
+    }
+    return ProxyCollections.retainAll(this, paramCollection);
+  }
+
+  protected Object writeReplace()
+    throws ObjectStreamException
+  {
+    if (isDelayLoad()) {
+        load();
+    }
+    return Proxies.writeReplace(this, true);
+  }
+
+  public boolean isDelayLoad() {
+      return ProxyCollections.isDelayed(this);
+    }
+  
+  @Override
+  public Object get(int location) {
+    if (!_directAccess && isDelayLoad()) {
+        load();
+    }
+    return super.get(location);
+  }
+
+
+@Override
+  public int indexOf(Object object) {
+    if (!_directAccess && isDelayLoad()) {
+        load();
+    }
+    return super.indexOf(object);
+  }
+
+  @Override
+  public int lastIndexOf(Object object) {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+    return super.lastIndexOf(object);
+  }
+
+  @Override
+  public List subList(int start, int end) {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+      return super.subList(start, end);
+  }
+
+  @Override
+  public boolean contains(Object object) {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+      return super.contains(object);
+  }
+
+  @Override
+  public boolean containsAll(Collection collection) {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+      return super.containsAll(collection);
+  }
+
+  @Override
+  public boolean isEmpty() {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+      return super.isEmpty();
+  }
+
+  @Override
+  public int size() {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+      return super.size();
+  }
+
+  @Override
+  public Object[] toArray() {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+      return super.toArray();
+  }
+
+  @Override
+  public Object[] toArray(Object[] array) {
+      if (!_directAccess && isDelayLoad()) {
+          load();
+      }
+      return super.toArray(array);
+  }
+
+  public boolean isDirectAccess() {
+      return _directAccess;
+  }
+  
+  public void setDirectAccess(boolean direct) {
+      _directAccess = direct;
+  }
+  
+  public BrokerFactory getBrokerFactory() {
+      return _brokerFactory;
+  }
+
+  @Override
+  public void load() {
+      ProxyCollections.loadCollection(this);
+  }
+
+  @Override
+  public Broker getBroker() {
+    if (_broker == null || _broker.isClosed()) {
+        if (_brokerFactory != null) {
+            _broker = _brokerFactory.newBroker();
+        }
+    }
+    return _broker;
+  }
+  
+  @Override
+  public void closeBroker() {
+      if (_broker != null && !_broker.isClosed()) {
+          _broker.setAutoDetach(AutoDetach.DETACH_CLOSE);
+          _broker.close();
+          _broker = null;
+      }
+  }
+
+  @Override
+  public OpenJPAStateManager getOwnerStateManager() {
+    return _ownerSm;
+  }
+}

openjpa-kernel/src/main/java/org/apache/openjpa/util/DelayedCollectionChangeTrackerImpl.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.    
+ */
+package org.apache.openjpa.util;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * A collection change tracker used by delay loaded collections.
+ *
+ * @nojavadoc
+ */
+public class DelayedCollectionChangeTrackerImpl
+    extends CollectionChangeTrackerImpl {
+
+    public DelayedCollectionChangeTrackerImpl(Collection coll, boolean dups,
+            boolean order,boolean autoOff) {
+        super(coll, dups, order, autoOff);
+    }
+
+    protected void add(Object elem) {
+        if (rem == null || !rem.remove(elem)) {
+            if (add == null) {
+                if (_dups || _order)
+                    add = new ArrayList();
+                else
+                    add = newSet();
+            }
+            add.add(elem);
+        } else {
+            if (change == null)
+                change = newSet();
+            change.add(elem);
+        }
+    }
+
+    protected void remove(Object elem) {
+        if (add == null || !add.remove(elem)) {
+            if (rem == null)
+                rem = newSet();
+            rem.add(elem);
+        }
+    }
+
+    protected void change(Object elem) {
+        throw new InternalException();
+    }
+}

openjpa-kernel/src/main/java/org/apache/openjpa/util/DelayedProxy.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.openjpa.util;
+
+import org.apache.openjpa.kernel.Broker;
+import org.apache.openjpa.kernel.OpenJPAStateManager;
+
+public interface DelayedProxy {
+    
+    void load();
+
+    boolean isDirectAccess();
+    
+    void setDirectAccess(boolean direct);
+    
+    Broker getBroker();
+
+    void closeBroker();
+
+    OpenJPAStateManager getOwnerStateManager();
+    
+    OpenJPAStateManager getDelayedOwner();
+    
+    int getDelayedField();
+}

openjpa-kernel/src/main/java/org/apache/openjpa/util/ProxyCollections.java

@@ -18,12 +18,18 @@
  */
 package org.apache.openjpa.util;
 
+import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.ListIterator;
 
+import org.apache.openjpa.kernel.Broker;
+import org.apache.openjpa.kernel.BrokerImpl;
+import org.apache.openjpa.kernel.DetachedValueStateManager;
+import org.apache.openjpa.kernel.OpenJPAStateManager;
+import org.apache.openjpa.kernel.StateManagerImpl;
+
 /**
  * Utility methods used by collection proxies.
  *
@@ -54,7 +60,10 @@ public class ProxyCollections
      */
     public static void beforeAdd(ProxyCollection coll, Object value) {
         assertAllowedType(value, coll.getElementType());
-        dirty(coll, false);
+        // Must only dirty the collection outside of a delayed load
+        if (!isDirectAccess(coll)) {
+            dirty(coll, false);
+        }
     }
 
     /**
@@ -65,8 +74,11 @@ public class ProxyCollections
      */
     public static boolean afterAdd(ProxyCollection coll, Object value, 
         boolean added) {
-        if (added && coll.getChangeTracker() != null)
+        if (!isDirectAccess(coll) && added && coll.getChangeTracker() != null) {
+            setDirectAccess(coll,true);
             ((CollectionChangeTracker) coll.getChangeTracker()).added(value);
+            setDirectAccess(coll,false);
+        }
         return added;
     }
 
@@ -139,7 +151,7 @@ public class ProxyCollections
      */
     public static boolean addAll(ProxyCollection coll, Collection values) {
         boolean added = false;
-        for (Iterator itr = values.iterator(); itr.hasNext();)
+        for (Iterator<?> itr = values.iterator(); itr.hasNext();)
             added |= coll.add(itr.next());
         return added;
     }
@@ -149,7 +161,7 @@ public class ProxyCollections
      */
     public static void beforeClear(ProxyCollection coll) {
         dirty(coll, true);
-        for (Iterator itr = coll.iterator(); itr.hasNext();)
+        for (Iterator<?> itr = coll.iterator(); itr.hasNext();)
             removed(coll, itr.next(), false);
     }
 
@@ -304,7 +316,10 @@ public class ProxyCollections
      * Call before invoking {@link Collection#remove} on super.
      */
     public static void beforeRemove(ProxyCollection coll, Object o) {
-        dirty(coll, false);
+        // Must only dirty the collection outside of a delayed load
+        if (!isDirectAccess(coll)) {
+            dirty(coll, false);
+        }
     }
 
     /**
@@ -315,14 +330,40 @@ public class ProxyCollections
      */ 
     public static boolean afterRemove(ProxyCollection coll, Object o, 
         boolean removed){
-        if (!removed)
+        boolean isDelayed = isDelayed(coll);
-            return false;
+        boolean direct = isDirectAccess(coll);
-        if (coll.getChangeTracker() != null)
+        if (!isDelayed) {
+            if (!removed)
+                return false;
+        }
+        if (!direct && coll.getChangeTracker() != null) {
+            // switch on direct access to prevent the removed op from 
+            // inadvertently loading the collection
+            setDirectAccess(coll, true);
             ((CollectionChangeTracker) coll.getChangeTracker()).removed(o);
-        removed(coll, o, false);
+            setDirectAccess(coll, false);
+        }
+        if (!isDelayed) {
+            removed(coll, o, false);
+        }
         return true;
     }
 
+    private static boolean isDirectAccess(ProxyCollection coll) {
+        if (coll instanceof DelayedProxy) {
+            DelayedProxy dpxy = (DelayedProxy)coll;
+            return dpxy.isDirectAccess();
+        }
+        return false;
+    }
+    
+    private static void setDirectAccess(ProxyCollection coll, boolean direct) {
+        if (coll instanceof DelayedProxy) {
+            DelayedProxy dpxy = (DelayedProxy)coll;
+            dpxy.setDirectAccess(direct);
+        }
+    }
+
     /**
      * Call before invoking {@link Vector#removeElement} on super.
      */
@@ -400,9 +441,9 @@ public class ProxyCollections
     /**
      * Override for {@link Collection#removeAll}.
      */
-    public static boolean removeAll(ProxyCollection coll, Collection vals) {
+    public static boolean removeAll(ProxyCollection coll, Collection<?> vals) {
         boolean removed = false;
-        for (Iterator itr = vals.iterator(); itr.hasNext();)
+        for (Iterator<?> itr = vals.iterator(); itr.hasNext();)
             removed |= coll.remove(itr.next());
         return removed;
     }
@@ -410,9 +451,9 @@ public class ProxyCollections
     /**
      * Override for {@link Collection#retainAll}.
      */
-    public static boolean retainAll(ProxyCollection coll, Collection vals) {
+    public static boolean retainAll(ProxyCollection coll, Collection<?> vals) {
         int size = coll.size();
-        for (Iterator itr = coll.iterator(); itr.hasNext();)
+        for (Iterator<?> itr = coll.iterator(); itr.hasNext();)
             if (!vals.contains(itr.next()))
                 itr.remove();
         return coll.size() < size;
@@ -469,4 +510,110 @@ public class ProxyCollections
     public static interface ProxyListIterator 
         extends ProxyIterator, ListIterator {
     }
+
+    public static void loadCollection(ProxyCollection proxy) {
+        loadCollection(proxy, false);
+    }
+
+    public static void loadCollection(ProxyCollection proxy, boolean detaching) {
+        if (!isDelayed(proxy)) {
+            return;
+        }
+        DelayedProxy dProxy = (DelayedProxy)proxy;
+        if (dProxy.isDirectAccess()) {
+            return;
+        }
+        boolean state[] = new boolean[2];
+        try {
+            dProxy.setDirectAccess(true);
+            state = checkState(proxy);
+            boolean tracking = false;
+            ChangeTracker ct = proxy.getChangeTracker();
+            Collection<?> added = null;
+            Collection<?> removed = null;
+            if (ct != null && ct.isTracking() ) {
+                if (!ct.getAdded().isEmpty()) {
+                    added = new ArrayList(ct.getAdded());
+                }
+                if (!ct.getRemoved().isEmpty()) {
+                    removed = new ArrayList(ct.getRemoved());
+                }
+                tracking = true;
+                ct.stopTracking();
+            }
+            if (proxy.size() > 0) {
+                proxy.clear();
+            }
+            dProxy.getDelayedOwner().loadDelayedField(dProxy.getDelayedField());
+            if (!detaching && tracking && !ct.isTracking()) {
+                ct.startTracking();
+            }
+            // add new elements
+            if (added != null && added.size() > 0) {
+                dProxy.setDirectAccess(false);
+                proxy.addAll(added);
+                added.clear();
+            }
+            // purge removed elements
+            if (removed != null && removed.size() > 0) {
+                dProxy.setDirectAccess(false);
+                proxy.removeAll(removed);
+                removed.clear();
+            }
+        } finally {
+            dProxy.setDirectAccess(false);
+            if (state[0]) {
+                dProxy.closeBroker();
+            }
+            if (state[1]) {
+                clearStateManager(proxy);
+            }
+        }
+    }
+    
+    public static boolean isDelayed(ProxyCollection proxy) {
+        if (proxy instanceof DelayedProxy) {
+            DelayedProxy dProxy = (DelayedProxy)proxy;
+            OpenJPAStateManager sm = dProxy.getDelayedOwner();
+            return (sm != null &&
+                    sm.isDelayed(dProxy.getDelayedField()));
+        }
+        return false;
+    }
+    
+    private static boolean[] checkState(ProxyCollection proxy) {
+        boolean[] state = new boolean[2];
+        DelayedProxy dProxy = (DelayedProxy)proxy;
+
+        OpenJPAStateManager sm = dProxy.getDelayedOwner();
+        if (sm != null) {
+            // If the broker assigned to this proxy is null, closed or no longer
+            // manages the pc, produce a new one
+            Broker broker = sm.getContext().getBroker();
+            if (broker == null || broker.isClosed() 
+                || (!broker.isClosed() && !broker.isPersistent(sm.getPersistenceCapable()))) {
+                state[0] = true;
+                broker = dProxy.getBroker();
+                ((StateManagerImpl)sm).setBroker((BrokerImpl)broker);
+            }
+            if (sm.getPersistenceCapable().pcGetStateManager() == null) {
+                state[1] = true;
+                if (dProxy.getOwnerStateManager() != null) {
+                    sm.getPersistenceCapable().pcReplaceStateManager(dProxy.getOwnerStateManager());
+                    ((StateManagerImpl)dProxy.getOwnerStateManager()).setBroker((BrokerImpl)broker);
+                } else {
+                    sm.getPersistenceCapable().pcReplaceStateManager(
+                            new DetachedValueStateManager(sm.getPersistenceCapable(), sm.getContext()));
+                }
+            }
+        }
+        return state;
+    }
+    
+    private static void clearStateManager(ProxyCollection proxy) {
+        OpenJPAStateManager sm = proxy.getOwner();
+        if (sm != null) {
+            sm.getPersistenceCapable().pcReplaceStateManager(null);
+        }
+    }
 }

openjpa-kernel/src/main/java/org/apache/openjpa/util/ProxyManager.java

@@ -113,4 +113,15 @@ public interface ProxyManager {
      * @since 0.2.5
      */
     public Proxy newCustomProxy (Object obj, boolean autoOff);
+    
+    /**
+     * Returns whether this proxy manager is enabled for delayed collection
+     * loading.  Delayed collection loading provides the ability to do simple,
+     * non-indexed add or remove operations on a lazy collection without 
+     * loading the collection.  The collection is loaded when necessary, such
+     * as iteration, indexed operations, isEmpty, or size.
+     *  
+     * @since 2.2.1
+     */
+    public boolean getDelayCollectionLoading();
 }

openjpa-kernel/src/main/java/org/apache/openjpa/util/ProxyManagerImpl.java

@@ -98,6 +98,7 @@ public class ProxyManagerImpl
     private final Map _proxies = new NullSafeConcurrentHashMap();
     private boolean _trackChanges = true;
     private boolean _assertType = false;
+    private boolean _delayedCollectionLoading = false;
 
     public ProxyManagerImpl() {
         _unproxyable.add(TimeZone.class.getName());
@@ -138,7 +139,26 @@ public class ProxyManagerImpl
     public void setAssertAllowedType(boolean assertType) {
         _assertType = assertType;
     }
-
+    
+    /**
+     * Whether loading of collections should be delayed until an operation
+     * is performed that requires them to be loaded.  This property only
+     * applies to proxies that implement java.util.Collection (ie. not arrays
+     * or maps).  Defaults to false.
+     * @return 
+     */
+    public boolean getDelayCollectionLoading() {
+        return _delayedCollectionLoading;
+    }
+    
+    /**
+     * Whether loading of collections should be delayed until an operation
+     * is performed that requires them to be loaded.  Defaults to false.
+     */
+    public void setDelayCollectionLoading(boolean delay) {
+        _delayedCollectionLoading = delay;
+    }
+       
     /**
      * Return a mutable view of class names we know cannot be proxied  
      * correctly by this manager.
@@ -477,6 +497,9 @@ public class ProxyManagerImpl
      */
     protected Class loadBuildTimeProxy(Class type, ClassLoader loader) {
         try {
+            if (_delayedCollectionLoading && type.equals(java.util.ArrayList.class)) {
+                return org.apache.openjpa.util.DelayedArrayListProxy.class;
+            }
             return Class.forName(getProxyClassName(type, false), true, loader);
         } catch (Throwable t) {
             return null;