Enforce a hard limit on meta size

This commit is contained in:
James Agnew 2017-06-30 09:58:32 -04:00
parent d626c58067
commit 28a5b92fe2
7 changed files with 225 additions and 145 deletions

View File

@ -26,52 +26,21 @@ import static org.apache.commons.lang3.StringUtils.trim;
import java.io.UnsupportedEncodingException;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import javax.persistence.EntityManager;
import javax.persistence.NoResultException;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
import javax.persistence.Tuple;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.persistence.*;
import javax.persistence.criteria.*;
import javax.xml.stream.events.Characters;
import javax.xml.stream.events.XMLEvent;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.*;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.hl7.fhir.dstu3.model.Bundle.HTTPVerb;
import org.hl7.fhir.dstu3.model.IdType;
import org.hl7.fhir.dstu3.model.StringType;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseBundle;
import org.hl7.fhir.instance.model.api.IBaseCoding;
import org.hl7.fhir.instance.model.api.IBaseExtension;
import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
import org.hl7.fhir.instance.model.api.IBaseReference;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IDomainResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.instance.model.api.IPrimitiveType;
import org.hl7.fhir.instance.model.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.PlatformTransactionManager;
@ -81,89 +50,31 @@ import com.google.common.collect.Sets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.context.RuntimeChildResourceDefinition;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.dao.data.IResourceHistoryTableDao;
import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao;
import ca.uhn.fhir.jpa.dao.data.ISearchDao;
import ca.uhn.fhir.jpa.dao.data.ISearchResultDao;
import ca.uhn.fhir.jpa.entity.BaseHasResource;
import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.entity.BaseTag;
import ca.uhn.fhir.jpa.entity.ForcedId;
import ca.uhn.fhir.jpa.entity.ResourceEncodingEnum;
import ca.uhn.fhir.jpa.entity.ResourceHistoryTable;
import ca.uhn.fhir.jpa.entity.ResourceHistoryTag;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamCoords;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamQuantity;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamString;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamUri;
import ca.uhn.fhir.jpa.entity.ResourceLink;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.ResourceTag;
import ca.uhn.fhir.jpa.entity.Search;
import ca.uhn.fhir.jpa.entity.SearchStatusEnum;
import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
import ca.uhn.fhir.jpa.entity.TagDefinition;
import ca.uhn.fhir.jpa.entity.TagTypeEnum;
import ca.uhn.fhir.context.*;
import ca.uhn.fhir.jpa.dao.data.*;
import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc;
import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider;
import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
import ca.uhn.fhir.jpa.util.DeleteConflict;
import ca.uhn.fhir.model.api.IQueryParameterAnd;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.IResource;
import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
import ca.uhn.fhir.model.api.Tag;
import ca.uhn.fhir.model.api.TagList;
import ca.uhn.fhir.model.api.*;
import ca.uhn.fhir.model.base.composite.BaseCodingDt;
import ca.uhn.fhir.model.base.composite.BaseResourceReferenceDt;
import ca.uhn.fhir.model.dstu.resource.BaseResource;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.InstantDt;
import ca.uhn.fhir.model.primitive.StringDt;
import ca.uhn.fhir.model.primitive.XhtmlDt;
import ca.uhn.fhir.model.primitive.*;
import ca.uhn.fhir.model.valueset.BundleEntryTransactionMethodEnum;
import ca.uhn.fhir.parser.DataFormatException;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.parser.LenientErrorHandler;
import ca.uhn.fhir.parser.*;
import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
import ca.uhn.fhir.rest.method.MethodUtil;
import ca.uhn.fhir.rest.method.QualifiedParamList;
import ca.uhn.fhir.rest.method.RestSearchParameterTypeEnum;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.StringAndListParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenAndListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.UriAndListParam;
import ca.uhn.fhir.rest.param.UriParam;
import ca.uhn.fhir.rest.method.*;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.server.Constants;
import ca.uhn.fhir.rest.server.IBundleProvider;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
import ca.uhn.fhir.rest.server.exceptions.*;
import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor;
import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor.ActionRequestDetails;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.util.CoverageIgnore;
import ca.uhn.fhir.util.FhirTerser;
import ca.uhn.fhir.util.OperationOutcomeUtil;
import ca.uhn.fhir.util.*;
public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
@ -634,6 +545,14 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
}
}
private Set<TagDefinition> getAllTagDefinitions(ResourceTable theEntity) {
HashSet<TagDefinition> retVal = Sets.newHashSet();
for (ResourceTag next : theEntity.getTags()) {
retVal.add(next.getTag());
}
return retVal;
}
protected DaoConfig getConfig() {
return myConfig;
}
@ -1000,14 +919,6 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
return changed;
}
private Set<TagDefinition> getAllTagDefinitions(ResourceTable theEntity) {
HashSet<TagDefinition> retVal = Sets.newHashSet();
for (ResourceTag next : theEntity.getTags()) {
retVal.add(next.getTag());
}
return retVal;
}
@SuppressWarnings("unchecked")
private <R extends IBaseResource> R populateResourceMetadataHapi(Class<R> theResourceType, BaseHasResource theEntity, boolean theForHistoryOperation, IResource res) {
R retVal = (R) res;
@ -1806,6 +1717,14 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
throw new ResourceVersionConflictException(firstMsg, oo);
}
protected void validateMetaCount(int theMetaCount) {
if (myConfig.getResourceMetaCountHardLimit() != null) {
if (theMetaCount > myConfig.getResourceMetaCountHardLimit()) {
throw new UnprocessableEntityException("Resource contains " + theMetaCount + " meta entries (tag/profile/security label), maximum is " + myConfig.getResourceMetaCountHardLimit());
}
}
}
/**
* This method is invoked immediately before storing a new resource, or an update to an existing resource to allow the DAO to ensure that it is valid for persistence. By default, checks for the
* "subsetted" tag and rejects resources which have it. Subclasses should call the superclass implementation to preserve this check.
@ -1817,15 +1736,23 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
*/
protected void validateResourceForStorage(T theResource, ResourceTable theEntityToSave) {
Object tag = null;
int totalMetaCount = 0;
if (theResource instanceof IResource) {
IResource res = (IResource) theResource;
TagList tagList = ResourceMetadataKeyEnum.TAG_LIST.get(res);
if (tagList != null) {
tag = tagList.getTag(Constants.TAG_SUBSETTED_SYSTEM, Constants.TAG_SUBSETTED_CODE);
}
totalMetaCount += tagList.size();
totalMetaCount += ResourceMetadataKeyEnum.PROFILES.get(res).size();
} else {
IAnyResource res = (IAnyResource) theResource;
tag = res.getMeta().getTag(Constants.TAG_SUBSETTED_SYSTEM, Constants.TAG_SUBSETTED_CODE);
totalMetaCount += res.getMeta().getTag().size();
totalMetaCount += res.getMeta().getProfile().size();
totalMetaCount += res.getMeta().getSecurity().size();
}
if (tag != null) {
@ -1835,6 +1762,8 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
String resName = getContext().getResourceDefinition(theResource).getName();
validateChildReferences(theResource, resName);
validateMetaCount(totalMetaCount);
}
protected static boolean isValidPid(IIdType theId) {

View File

@ -431,7 +431,6 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
private <MT extends IBaseMetaType> void doMetaAdd(MT theMetaAdd, BaseHasResource entity) {
List<TagDefinition> tags = toTagList(theMetaAdd);
//@formatter:off
for (TagDefinition nextDef : tags) {
boolean hasTag = false;
@ -454,8 +453,9 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
}
}
}
//@formatter:on
validateMetaCount(entity.getTags().size());
myEntityManager.merge(entity);
}

View File

@ -103,13 +103,19 @@ public class DaoConfig {
* update setter javadoc if default changes
*/
private boolean myIndexContainedResources = true;
private List<IServerInterceptor> myInterceptors;
/**
* update setter javadoc if default changes
*/
private int myMaximumExpansionSize = 5000;
private int myMaximumSearchResultCountInTransaction = DEFAULT_MAXIMUM_SEARCH_RESULT_COUNT_IN_TRANSACTION;
private ResourceEncodingEnum myResourceEncoding = ResourceEncodingEnum.JSONC;
/**
* update setter javadoc if default changes
*/
private Integer myResourceMetaCountHardLimit = 1000;
private Long myReuseCachedSearchResultsForMillis = DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS;
private boolean mySchedulingDisabled;
private boolean mySubscriptionEnabled;
@ -121,7 +127,6 @@ public class DaoConfig {
private boolean mySuppressUpdatesWithNoChange = true;
private Set<String> myTreatBaseUrlsAsLocal = new HashSet<String>();
private Set<String> myTreatReferencesAsLogical = new HashSet<String>(DEFAULT_LOGICAL_BASE_URLS);
/**
* Add a value to the {@link #setTreatReferencesAsLogical(Set) logical references list}.
*
@ -135,7 +140,6 @@ public class DaoConfig {
}
myTreatReferencesAsLogical.add(theTreatReferencesAsLogical);
}
/**
* When a code system is added that contains more than this number of codes,
* the code system will be indexed later in an incremental process in order to
@ -244,6 +248,21 @@ public class DaoConfig {
return myResourceEncoding;
}
/**
* If set, an individual resource will not be allowed to have more than the
* given number of tags, profiles, and security labels (the limit is for the combined
* total for all of these things on an individual resource).
* <p>
* If set to <code>null</code>, no limit will be applied.
* </p>
* <p>
* The default value for this setting is 1000.
* </p>
*/
public Integer getResourceMetaCountHardLimit() {
return myResourceMetaCountHardLimit;
}
/**
* If set to a non {@literal null} value (default is {@link #DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS non null})
* if an identical search is requested multiple times within this window, the same results will be returned
@ -376,8 +395,6 @@ public class DaoConfig {
* and other FHIR features may not behave as expected when referential integrity is not
* preserved. Use this feature with caution.
* </p>
*
* @return
*/
public boolean isEnforceReferentialIntegrityOnDelete() {
return myEnforceReferentialIntegrityOnDelete;
@ -697,6 +714,21 @@ public class DaoConfig {
myResourceEncoding = theResourceEncoding;
}
/**
* If set, an individual resource will not be allowed to have more than the
* given number of tags, profiles, and security labels (the limit is for the combined
* total for all of these things on an individual resource).
* <p>
* If set to <code>null</code>, no limit will be applied.
* </p>
* <p>
* The default value for this setting is 1000.
* </p>
*/
public void setResourceMetaCountHardLimit(Integer theResourceMetaCountHardLimit) {
myResourceMetaCountHardLimit = theResourceMetaCountHardLimit;
}
/**
* If set to a non {@literal null} value (default is {@link #DEFAULT_REUSE_CACHED_SEARCH_RESULTS_FOR_MILLIS non null})
* if an identical search is requested multiple times within this window, the same results will be returned

View File

@ -22,21 +22,13 @@ package ca.uhn.fhir.jpa.entity;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Embeddable;
import javax.persistence.Entity;
import javax.persistence.ForeignKey;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.SequenceGenerator;
import javax.persistence.Table;
import javax.persistence.*;
@Embeddable
@Entity
@Table(name = "HFJ_HISTORY_TAG")
@Table(name = "HFJ_HISTORY_TAG", uniqueConstraints= {
@UniqueConstraint(name="IDX_RESHISTTAG_TAGID", columnNames= {"RES_VER_PID","TAG_ID"})
})
public class ResourceHistoryTag extends BaseTag implements Serializable {
private static final long serialVersionUID = 1L;

View File

@ -19,23 +19,15 @@ package ca.uhn.fhir.jpa.entity;
* limitations under the License.
* #L%
*/
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.ForeignKey;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.SequenceGenerator;
import javax.persistence.Table;
import javax.persistence.*;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
@Entity
@Table(name = "HFJ_RES_TAG")
@Table(name = "HFJ_RES_TAG", uniqueConstraints= {
@UniqueConstraint(name="IDX_RESTAG_TAGID", columnNames= {"RES_ID","TAG_ID"})
})
public class ResourceTag extends BaseTag {
private static final long serialVersionUID = 1L;

View File

@ -33,11 +33,10 @@ import org.hl7.fhir.dstu3.model.Resource;
import org.hl7.fhir.dstu3.model.UriType;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.AfterClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.*;
import org.mockito.ArgumentCaptor;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.dao.SearchParameterMap;
import ca.uhn.fhir.model.primitive.InstantDt;
import ca.uhn.fhir.rest.api.MethodOutcome;
@ -123,6 +122,137 @@ public class FhirResourceDaoDstu3UpdateTest extends BaseJpaDstu3Test {
}
@After
public void afterResetDao() {
myDaoConfig.setResourceMetaCountHardLimit(new DaoConfig().getResourceMetaCountHardLimit());
}
@Test
public void testHardMetaCapIsEnforcedOnCreate() {
myDaoConfig.setResourceMetaCountHardLimit(3);
IIdType id;
{
Patient patient = new Patient();
patient.getMeta().addTag().setSystem("http://foo").setCode("1");
patient.getMeta().addTag().setSystem("http://foo").setCode("2");
patient.getMeta().addTag().setSystem("http://foo").setCode("3");
patient.getMeta().addTag().setSystem("http://foo").setCode("4");
patient.setActive(true);
try {
id = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
fail();
} catch (UnprocessableEntityException e) {
assertEquals("Resource contains 4 meta entries (tag/profile/security label), maximum is 3", e.getMessage());
}
}
}
@Test
public void testHardMetaCapIsEnforcedOnMetaAdd() {
myDaoConfig.setResourceMetaCountHardLimit(3);
IIdType id;
{
Patient patient = new Patient();
patient.setActive(true);
id = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
}
{
Meta meta = new Meta();
meta.addTag().setSystem("http://foo").setCode("1");
meta.addTag().setSystem("http://foo").setCode("2");
meta.addTag().setSystem("http://foo").setCode("3");
meta.addTag().setSystem("http://foo").setCode("4");
try {
myPatientDao.metaAddOperation(id, meta, null);
fail();
} catch (UnprocessableEntityException e) {
assertEquals("Resource contains 4 meta entries (tag/profile/security label), maximum is 3", e.getMessage());
}
}
}
@Test
public void testDuplicateTagsOnAddTagsIgnored() {
IIdType id;
{
Patient patient = new Patient();
patient.setActive(true);
id = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
}
Meta meta = new Meta();
meta.addTag().setSystem("http://foo").setCode("bar").setDisplay("Val1");
meta.addTag().setSystem("http://foo").setCode("bar").setDisplay("Val2");
meta.addTag().setSystem("http://foo").setCode("bar").setDisplay("Val3");
myPatientDao.metaAddOperation(id, meta, null);
// Do a read
{
Patient patient = myPatientDao.read(id, mySrd);
List<Coding> tl = patient.getMeta().getTag();
assertEquals(1, tl.size());
assertEquals("http://foo", tl.get(0).getSystem());
assertEquals("bar", tl.get(0).getCode());
}
}
@Test
public void testDuplicateTagsOnUpdateIgnored() {
IIdType id;
{
Patient patient = new Patient();
patient.setActive(true);
id = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
}
{
Patient patient = new Patient();
patient.setId(id);
patient.setActive(true);
patient.getMeta().addTag().setSystem("http://foo").setCode("bar").setDisplay("Val1");
patient.getMeta().addTag().setSystem("http://foo").setCode("bar").setDisplay("Val2");
patient.getMeta().addTag().setSystem("http://foo").setCode("bar").setDisplay("Val3");
myPatientDao.update(patient, mySrd).getId().toUnqualifiedVersionless();
}
// Do a read on second version
{
Patient patient = myPatientDao.read(id, mySrd);
List<Coding> tl = patient.getMeta().getTag();
assertEquals(1, tl.size());
assertEquals("http://foo", tl.get(0).getSystem());
assertEquals("bar", tl.get(0).getCode());
}
// Do a read on first version
{
Patient patient = myPatientDao.read(id.withVersion("1"), mySrd);
List<Coding> tl = patient.getMeta().getTag();
assertEquals(0, tl.size());
}
Meta meta = new Meta();
meta.addTag().setSystem("http://foo").setCode("bar").setDisplay("Val1");
meta.addTag().setSystem("http://foo").setCode("bar").setDisplay("Val2");
meta.addTag().setSystem("http://foo").setCode("bar").setDisplay("Val3");
myPatientDao.metaAddOperation(id.withVersion("1"), meta, null);
// Do a read on first version
{
Patient patient = myPatientDao.read(id.withVersion("1"), mySrd);
List<Coding> tl = patient.getMeta().getTag();
assertEquals(1, tl.size());
assertEquals("http://foo", tl.get(0).getSystem());
assertEquals("bar", tl.get(0).getCode());
}
}
@Test
public void testMultipleUpdatesWithNoChangesDoesNotResultInAnUpdateForDiscreteUpdates() {

View File

@ -80,6 +80,11 @@
were not able to be automatically purged from the database after they were scheduled
for deletion. Thanks to Ravi Kuchi for reporting!
</action>
<action type="add">
Add an optional and configurable hard limit on the total number of meta items
(tags, profiles, and security labels) on an individual resource. The default
is 1000.
</action>
</release>
<release version="2.5" date="2017-06-08">
<action type="fix">