Filter all navigation properties before they are loaded (lazy or eager) into memory

Question

For future visitors: for EF6 you are probably better off using filters, for example via this project: https://github.com/jbogard/EntityFramework.Filters

In the applicati

Answer 1

After much research, I've finally found a way to achieve what I wanted. The gist of it is that I intercept materialized entities with an event handler on the object context, and then inject my custom collection class in every collection property that I can find (with reflection).

The most important part is intercepting the "DbCollectionEntry", the class responsible for loading related collection properties. By wiggling myself in between the entity and the DbCollectionEntry, I gain full control over what's loaded when and how. The only downside is that this DbCollectionEntry class has little to no public members, which requires me to use reflection to manipulate it.

Here is my custom collection class that implements ICollection and contains a reference to the appropriate DbCollectionEntry:

public class FilteredCollection  : ICollection where TEntity : Entity
{
    private readonly DbCollectionEntry _dbCollectionEntry;
    private readonly Func _compiledFilter;
    private readonly Expression> _filter;
    private ICollection _collection;
    private int? _cachedCount;

    public FilteredCollection(ICollection collection, DbCollectionEntry dbCollectionEntry)
    {
        _filter = entity => !entity.Deleted;
        _dbCollectionEntry = dbCollectionEntry;
        _compiledFilter = _filter.Compile();
        _collection = collection != null ? collection.Where(_compiledFilter).ToList() : null;
    }

    private ICollection Entities
    {
        get
        {
            if (_dbCollectionEntry.IsLoaded == false && _collection == null)
            {
                IQueryable query = _dbCollectionEntry.Query().Cast().Where(_filter);
                _dbCollectionEntry.CurrentValue = this;
                _collection = query.ToList();

                object internalCollectionEntry =
                    _dbCollectionEntry.GetType()
                        .GetField("_internalCollectionEntry", BindingFlags.NonPublic | BindingFlags.Instance)
                        .GetValue(_dbCollectionEntry);
                object relatedEnd =
                    internalCollectionEntry.GetType()
                        .BaseType.GetField("_relatedEnd", BindingFlags.NonPublic | BindingFlags.Instance)
                        .GetValue(internalCollectionEntry);
                relatedEnd.GetType()
                    .GetField("_isLoaded", BindingFlags.NonPublic | BindingFlags.Instance)
                    .SetValue(relatedEnd, true);
            }
            return _collection;
        }
    }

    #region ICollection Members

    void ICollection.Add(TEntity item)
    {
        if(_compiledFilter(item))
            Entities.Add(item);
    }

    void ICollection.Clear()
    {
        Entities.Clear();
    }

    Boolean ICollection.Contains(TEntity item)
    {
        return Entities.Contains(item);
    }

    void ICollection.CopyTo(TEntity[] array, Int32 arrayIndex)
    {
        Entities.CopyTo(array, arrayIndex);
    }

    Int32 ICollection.Count
    {
        get
        {
            if (_dbCollectionEntry.IsLoaded)
                return _collection.Count;
            return _dbCollectionEntry.Query().Cast().Count(_filter);
        }
    }

    Boolean ICollection.IsReadOnly
    {
        get
        {
            return Entities.IsReadOnly;
        }
    }

    Boolean ICollection.Remove(TEntity item)
    {
        return Entities.Remove(item);
    }

    #endregion

    #region IEnumerable Members

    IEnumerator IEnumerable.GetEnumerator()
    {
        return Entities.GetEnumerator();
    }

    #endregion

    #region IEnumerable Members

    IEnumerator IEnumerable.GetEnumerator()
    {
        return ( ( this as IEnumerable ).GetEnumerator() );
    }

    #endregion
}

If you skim through it, you'll find that the most important part is the "Entities" property, which will lazy load the actual values. In the constructor of the FilteredCollection I pass an optional ICollection for scenario's where the collection is already eagerly loaded.

Of course, we still need to configure Entity Framework so that our FilteredCollection is used everywhere where there are collection properties. This can be achieved by hooking into the ObjectMaterialized event of the underlying ObjectContext of Entity Framework:

(this as IObjectContextAdapter).ObjectContext.ObjectMaterialized +=
    delegate(Object sender, ObjectMaterializedEventArgs e)
    {
        if (e.Entity is Entity)
        {
            var entityType = e.Entity.GetType();
            IEnumerable collectionProperties;
            if (!CollectionPropertiesPerType.TryGetValue(entityType, out collectionProperties))
            {
                CollectionPropertiesPerType[entityType] = (collectionProperties = entityType.GetProperties()
                    .Where(p => p.PropertyType.IsGenericType && typeof(ICollection<>) == p.PropertyType.GetGenericTypeDefinition()));
            }
            foreach (var collectionProperty in collectionProperties)
            {
                var collectionType = typeof(FilteredCollection<>).MakeGenericType(collectionProperty.PropertyType.GetGenericArguments());
                DbCollectionEntry dbCollectionEntry = Entry(e.Entity).Collection(collectionProperty.Name);
                dbCollectionEntry.CurrentValue = Activator.CreateInstance(collectionType, new[] { dbCollectionEntry.CurrentValue, dbCollectionEntry });
            }
        }
    };

It all looks rather complicated, but what it does essentially is scan the materialized type for collection properties and change the value to a filtered collection. It also passes the DbCollectionEntry to the filtered collection so it can work its magic.

This covers the whole 'loading entities' part. The only downside so far is that eagerly loaded collection properties will still include the deleted entities, but they are filtered out in the 'Add' method of the FilterCollection class. This is an acceptable downside, although I have yet to do some testing on how this affects the SaveChanges() method.

Of course, this still leaves one issue: there is no automatic filtering on queries. If you want to fetch the gym members who did a workout in the past week, you want to exclude the deleted workouts automatically.

This is achieved through an ExpressionVisitor that automatically applies a '.Where(e => !e.Deleted)' filter to every IQueryable it can find in a given expression.

Here is the code:

public class DeletedFilterInterceptor: ExpressionVisitor
{
    public Expression> Filter { get; set; }

    public DeletedFilterInterceptor()
    {
        Filter = entity => !entity.Deleted;
    }

    protected override Expression VisitMember(MemberExpression ex)
    {
        return !ex.Type.IsGenericType ? base.VisitMember(ex) : CreateWhereExpression(Filter, ex) ?? base.VisitMember(ex);
    }

    private Expression CreateWhereExpression(Expression> filter, Expression ex)
    {
        var type = ex.Type;//.GetGenericArguments().First();
        var test = CreateExpression(filter, type);
        if (test == null)
            return null;
        var listType = typeof(IQueryable<>).MakeGenericType(type);
        return Expression.Convert(Expression.Call(typeof(Enumerable), "Where", new Type[] { type }, (Expression)ex, test), listType);
    }

    private LambdaExpression CreateExpression(Expression> condition, Type type)
    {
        var lambda = (LambdaExpression) condition;
        if (!typeof(Entity).IsAssignableFrom(type))
            return null;

        var newParams = new[] { Expression.Parameter(type, "entity") };
        var paramMap = lambda.Parameters.Select((original, i) => new { original, replacement = newParams[i] }).ToDictionary(p => p.original, p => p.replacement);
        var fixedBody = ParameterRebinder.ReplaceParameters(paramMap, lambda.Body);
        lambda = Expression.Lambda(fixedBody, newParams);

        return lambda;
    }
}

public class ParameterRebinder : ExpressionVisitor
{
    private readonly Dictionary _map;

    public ParameterRebinder(Dictionary map)
    {
        _map = map ?? new Dictionary();
    }

    public static Expression ReplaceParameters(Dictionary map, Expression exp)
    {
        return new ParameterRebinder(map).Visit(exp);
    }

    protected override Expression VisitParameter(ParameterExpression node)
    {
        ParameterExpression replacement;

        if (_map.TryGetValue(node, out replacement))
            node = replacement;

        return base.VisitParameter(node);
    }
}

I am running a bit short on time, so I'll get back to this post later with more details, but the gist of it is written down and for those of you eager to try everything out; I've posted the full test application here: https://github.com/amoerie/TestingGround

However, there might still be some errors, as this is very much a work in progress. The conceptual idea is sound though, and I expect it to fully function soon once I've refactored everything neatly and find the time to write some tests for this.