如何在foreach语句上优化EF性能

这是我的代码;如您所见,我正在为每一行保存更改,但是我想提高性能,因为每次我都有大量数据,例如50.000或100.000行或更多。这需要很多时间才能完成。

如何提高EF SaveChanges的性能?我尝试使用一些第三方库进行bulksave,bulkupdate,但数据库中未更新。这需要2个小时来更新50.000行。我想缩短这种方法的时间。

private void TransferOrders()
{
    using (var context = new BbsfDbContext())
    {
        context.DisableFilter(AbpDataFilters.MayHaveTenant);
        context.DisableFilter("LanguageSpecificFilter");

        var sapOrders = context.SapOrders
                               .Where(p => p.VBTYP != null && 
                                           p.VBTYP.ToLower() == OrderDocumentType && 
                                           p.IsRead == false)
                                //.Where(p => p.VBTYP != null && p.VBTYP.ToLower() == OrderDocumentType && p.Id == 3025)
                               .Where(p => !activeUsersOnly || context.Users.Where(u => u.Isactive).Select(a => a.MainVendor.SapCode).Contains(p.KUNNR))
                               .OrderBy(p => p.CreatedDate)
                               .ToList();

        if (sapOrders.Any())
        {
            foreach (var item in sapOrders)
            {
                try
                {
                    var order = context.Orders.FirstOrDefault(p => p.SapCode == item.VBELN);

                    var isExist = context.SapOrderDetails.Any(p => p.DOCNUM == item.DOCNUM);

                    if (isExist)
                    {
                        var salesOrganization = context.SalesOrganizations.FirstOrDefault(p => p.SapCode == item.VKORG);

                        if (salesOrganization == null)
                            continue;

                        var distributionChannel = context.DistributionChannels.FirstOrDefault(p => p.SapCode == item.VTWEG);

                        if (distributionChannel == null)
                            continue;

                        var salesDepartment = context.SalesDepartments.FirstOrDefault(p => p.SapCode == item.SPART);

                        if (salesDepartment == null)
                            continue;

                        var salesOffice = context.SalesOffices
                                                 .FirstOrDefault(p => p.SapCode == item.VKBUR &&
                                                     p.SalesOrganization.Id == salesOrganization.Id &&
                                                     p.DistributionChannel.Id == distributionChannel.Id &&
                                                     p.SalesDepartment.Id == salesDepartment.Id);
                        if (salesOffice == null)
                            continue;

                        var ordererCustomer = context.Customers
                                .FirstOrDefault(p => p.SapCode == item.KUNNR &&
                                                     p.SalesOrganization.Id == salesOrganization.Id &&
                                                     p.DistributionChannel.Id == distributionChannel.Id &&
                                                     p.SalesDepartment.Id == salesDepartment.Id &&
                                                     p.SalesOffice.Id == salesOffice.Id);

                        var recipientCustomer = context.Customers
                                .FirstOrDefault(p => p.SapCode == item.KUNWE &&
                                                     p.SalesOrganization.Id == salesOrganization.Id &&
                                                     p.DistributionChannel.Id == distributionChannel.Id &&
                                                     p.SalesDepartment.Id == salesDepartment.Id &&
                                                     p.SalesOffice.Id == salesOffice.Id);

                        if (recipientCustomer == null)
                            recipientCustomer = context.Customers
                                    .FirstOrDefault(p => p.SapCode == item.KUNWE &&
                                                         p.SalesOrganization.Id == salesOrganization.Id &&
                                                         p.DistributionChannel.Id == distributionChannel.Id &&
                                                         p.SalesDepartment.Id == salesDepartment.Id &&
                                                         p.SalesOffice == null);

                        if (ordererCustomer == null || recipientCustomer == null)
                            continue;

                        if (order == null)
                        {
                            order = new Order
                                {
                                    SapCode = item.VBELN,SapOrderDate = item.AUDAT,DocumentType = context.DocumentTypes.FirstOrDefault(p => p.SapCode == item.VBTYP),SalesDocument = context.SalesDocuments.FirstOrDefault(p => p.SapCode == item.AUART),BaseAmount = item.NETWR,TotalTax = item.MWSBT,Currency = context.CurrencyDefinitions.FirstOrDefault(p => p.SapCode == item.WAERK),SalesOrganization = salesOrganization,DistributionChannel = distributionChannel,SalesDepartment = salesDepartment,SalesGroup = context.SalesGroups.FirstOrDefault(p => p.SapCode == item.VKGRP && p.SalesOffice.Id == salesOffice.Id),SalesOffice = salesOffice,RequestedDeliveryDate = item.VDATU,SAsno = item.BSTNK,SASOrderDate = item.BSTDK ?? item.AUDAT,OrdererCustomer = ordererCustomer,RecipientCustomer = recipientCustomer,//PRSDT
                                    Status = OrderStatus.Approved,Type = OrderType.MainVendor,DeliveryAddress = context.CustomerAddressBooks.FirstOrDefault(p => p.MainVendor.Id == ordererCustomer.Id && p.SubVendor.Id == recipientCustomer.Id),CreationTime = DateTime.Now,LastModificationTime = DateTime.Now,CreatorUserId = context.Users.First(p => p.username == AbpUserBase.Adminusername).Id,LastModifierUserId = context.Users.First(p => p.username == AbpUserBase.Adminusername).Id,IsSubVendorOrder = false,IsSameDayDelivery = false,RepresentativeId = context.Users.First(p => p.username == AbpUserBase.Adminusername).Id
                                    //ProductionSite
                                    //RejectionReason =//todo:bu silinmeli iptal kalem bazında burada statu olmalı
                                };
                                var savedOrder = context.Orders.Add(order);
                                context.SaveChanges();

                                order.SAsno = BbsfConsts.KeasOrderNumberPrefix + savedOrder.Id;
                            }
                            else
                            {
                                order.SapOrderDate = item.AUDAT;
                                order.DocumentType = context.DocumentTypes.FirstOrDefault(p => p.SapCode == item.VBTYP);
                                order.SalesDocument = context.SalesDocuments.FirstOrDefault(p => p.SapCode == item.AUART);
                                order.BaseAmount = item.NETWR;
                                order.TotalTax = item.MWSBT;
                                order.Currency = context.CurrencyDefinitions.FirstOrDefault(p => p.SapCode == item.WAERK);
                                order.SalesOrganization = salesOrganization;
                                order.DistributionChannel = distributionChannel;
                                order.SalesDepartment = salesDepartment;
                                order.SalesGroup = context.SalesGroups.FirstOrDefault(p => p.SapCode == item.VKGRP && p.SalesOffice.Id == salesOffice.Id);
                                order.SalesOffice = salesOffice;
                                order.RequestedDeliveryDate = item.VDATU;
                                order.SAsno = BbsfConsts.KeasOrderNumberPrefix + order.Id;
                                //order.SASOrderDate = item.BSTDK.HasValue ? item.BSTDK : item.AUDAT;
                                order.OrdererCustomer = ordererCustomer;
                                order.RecipientCustomer = recipientCustomer;
                                //PRSDT
                                //order.Status = OrderStatus.Approved;
                                order.DeliveryAddress = context.CustomerAddressBooks.FirstOrDefault(p => p.MainVendor.Id == ordererCustomer.Id && p.SubVendor.Id == recipientCustomer.Id);
                                order.LastModifierUserId = context.Users.First(p => p.username == AbpUserBase.Adminusername).Id;
                                order.LastModificationTime = DateTime.Now;
                                //ProductionSite
                                //RejectionReason =//todo:bu silinmeli iptal kalem bazında burada statu olmalı
                            }
                        }
                        else
                        {
                            if (order != null)
                            {
                                var orderDetails = context.OrderDetails.Where(p => p.OrderId == order.Id).ToList();
                                orderDetails?.ForEach(p => context.OrderDetails.Remove(p));
                                context.SaveChanges();

                                context.Orders.Remove(order);
                                context.SaveChanges();
                            }
                        }

                        item.IsRead = true;
                        item.ModifiedDate = DateTime.Now;

                        context.SaveChanges();
                    }
                    catch (Exception ex)
                    {
                        logger.Error(ex,MethodBase.getcurrentMethod().Name + " Error During IDOCOperations " + ex.Message);
                        continue;
                    }
                }
            }
        }
    }
WIND888888 回答:如何在foreach语句上优化EF性能

乍一看,似乎可以将不同的动作组合在一起,然后在一个步骤中执行。

例如:

  • 创建所有必须用avg_over_time()更新的项目的列表,并一步一步地执行所有操作
  • 以同样的方式,创建一个包含所有要删除订单的列表,然后在最后一步执行

我不知道这是否适用于上下文和/或应用程序,只是一个想法

,

首先,检查全局和按属性级别的延迟加载功能的配置。

当您在sapOrders中执行ToList()时,您正在执行查询(并将结果加载到内存中),并且当您获得Orders,SalesOrganizations等时,您可能正在执行辅助查询...

看看this文章以改善循环。

另一方面,您可以在循环中使用异步查询和并行编程,因此,对于其中的每个非依赖任务,您都可以创建一个异步Task方法并同时运行它们。

Herehere是一些文章,希望对您有所帮助。

祝你好运!

,

如果您可以避免在循环中调用.SaveChanges(),而只是在最后执行此操作,那么您会更好,因为可以避免多次往返数据库。话虽这么说,如果您要遍历50,000个项目,您可能希望在某种程度上进行批量处理,也许每1,000个您调用一次。

不幸的是,您的复杂性是需要保存时创建订单的结果才能存储在另一列上。也许,如果您跟踪正在创建的订单,那么在进行批量保存时,是否可以立即对刚刚创建的每个订单和另外一个SASNo进行一组.BulkSaveChanges()的批量处理?>

对于以后的保存更改(在其中删除订单详细信息,保存,删除订单,保存),我看不到需要分多个步骤进行,但是我的EF生锈了,并且会抱怨。理想情况下,我将所有对.SaveChanges()的调用删除,然后每1000个批量操作执行一次。

假设BulkSaveChanges可以处理所有这些,以上将大大减少DB网络调用的数量。基本上,我的目标是在下面,但是最终,如果没有EF,这可能会做得更好/更快。

using (var context = new BbsfDbContext())
{
    var sapOrders = ...;
    var ordersCreated = new List<..>(); // might wanna initialized this with a size if you have a rough gauge on what % will need creation of loop

    //if (sapOrders.Any()) // not needed
    //{
        foreach (var item in sapOrders.Select((x,index) => new { x,index }))
        {
            try
            {
                var order = ...;
                var isExist = ...;

                if (isExist)
                {
                    // ...

                    if (order == null)
                    {
                        order = new Order { ... };
                            var savedOrder = context.Orders.Add(order);
                            //context.SaveChanges();

                            //order.SASNo = BbsfConsts.KeasOrderNumberPrefix + savedOrder.Id;
                            ordersCreated.Add(order);
                        }
                        else
                        {
                            // Do updates
                            // ...
                        }
                    }
                    else
                    {
                        //if (order != null) // shouldn't need this
                        //{
                            var orderDetails = context.OrderDetails.Where(p => p.OrderId == order.Id).ToList();
                            orderDetails?.ForEach(p => context.OrderDetails.Remove(p));
                            //context.SaveChanges();

                            context.Orders.Remove(order);
                            //context.SaveChanges();
                        //}
                    }

                    // ...

                    if (index % 1000 == 0)
                    {
                        context.BulkSaveChanges(); // bulk save of 1000 loops of changes

                        foreach (var orderCreated in ordersCreated)
                        {
                            orderCreated.SASNo = BbsfConsts.KeasOrderNumberPrefix + savedOrder.Id;
                        }
                        context.BulkSaveChanges(); // bulk save of x num of SASNo sets
                    }
                }
                catch (Exception ex)
                {
                    // ...
                }
            }
        }
    }
}
本文链接:https://www.f2er.com/3146599.html

大家都在问