feature 代码提交 解决导入慢的问题

zhsw-common/src/main/java/com/rongwei/zhsw/system/config/TtlThreadPoolConfig.java

@@ -22,9 +22,9 @@ public class TtlThreadPoolConfig {
     @Bean(name = "zhswThreadPool")
     public Executor ttlTaskExecutor() {
         ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
-        executor.setCorePoolSize(4);
-        executor.setMaxPoolSize(8);
-        executor.setQueueCapacity(16);
+        executor.setCorePoolSize(16);
+        executor.setMaxPoolSize(32);
+        executor.setQueueCapacity(32);
         executor.setThreadNamePrefix("zhsw-");
         executor.setTaskDecorator(new TtlTaskDecorator());
         // executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());

zhsw-common/src/main/java/com/rongwei/zhsw/system/dao/SwWaterUsageEntryDao.java

@@ -4,6 +4,9 @@ package com.rongwei.zhsw.system.dao;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.rongwe.zhsw.system.domain.SwWaterUsageEntryDo;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
 
 /**
  * 抄表登记(SwWaterUsageEntry)表数据库访问层
@@ -14,5 +17,6 @@ import org.apache.ibatis.annotations.Mapper;
 @Mapper
 public interface SwWaterUsageEntryDao extends BaseMapper<SwWaterUsageEntryDo> {
 
+    void  ListBatchSave(@Param("swWaterUsageEntryDos") List<SwWaterUsageEntryDo> swWaterUsageEntryDos,@Param("dsKey")String dsKey);
 }
 

zhsw-common/src/main/java/com/rongwei/zhsw/system/importListener/MeterReadingRecordListener.java

@@ -119,6 +119,23 @@ public class MeterReadingRecordListener extends AnalysisEventListener<ImportMete
         // 获取户号信息
         Lists.partition(accountNumList, 5000).forEach(accountNum -> {
             allAccountData.addAll(swUserManagementService.list(new LambdaQueryWrapper<SwUserManagementDo>()
+                    .select(SwUserManagementDo::getId,
+                            SwUserManagementDo::getUsernumber,
+                            SwUserManagementDo::getMetermaxvalue,
+                            SwUserManagementDo::getWaterprice,
+                            SwUserManagementDo::getVillagename,
+                            SwUserManagementDo::getVolumeno,
+                            SwUserManagementDo::getUsernumber,
+                            SwUserManagementDo::getUsername,
+                            SwUserManagementDo::getUsertype,
+                            SwUserManagementDo::getUsertypeid,
+                            SwUserManagementDo::getAddress,
+                            SwUserManagementDo::getExemptionamount,
+                            SwUserManagementDo::getExemptionwater,
+                            SwUserManagementDo::getWatertype,
+                            SwUserManagementDo::getLastmeterreaddate,
+                            SwUserManagementDo::getLastmeterreading
+                    )
                     .eq(BaseDo::getDeleted, "0")
                     .in(SwUserManagementDo::getUsernumber, accountNumList)));
         });

zhsw-common/src/main/java/com/rongwei/zhsw/system/service/impl/BillGenerationServiceImpl.java

@@ -19,7 +19,6 @@ import org.springframework.stereotype.Component;
 import org.springframework.transaction.TransactionDefinition;
 import org.springframework.transaction.TransactionStatus;
 import org.springframework.transaction.support.DefaultTransactionDefinition;
-import org.springframework.web.context.request.RequestContextHolder;
 
 import java.math.BigDecimal;
 import java.math.RoundingMode;
@@ -77,8 +76,9 @@ public class BillGenerationServiceImpl {
 
     public void generateBill(List<SwWaterUsageEntryDo> swWaterUsageEntryDoList) {
         String dsKey = ContextHolder.getValue("dsKey");
-        log.info("dsKey:{}",  dsKey);
+        log.info("dsKey:{}", dsKey);
         log.debug("需要生成账单的抄表记录为：{}", swWaterUsageEntryDoList);
+        long asyncTasksStart = System.currentTimeMillis();
         // 对数据按照抄表日期排序
         List<SwWaterUsageEntryDo> collect = swWaterUsageEntryDoList.stream()
                 .sorted(Comparator.comparing(SwWaterUsageEntryDo::getCurrentreadingdate))
@@ -118,6 +118,7 @@ public class BillGenerationServiceImpl {
 
         SwUserManagementDo swUserManagementDo;
         SwBillManagementUnpaidDo swBillManagementUnpaidDo = null;
+        SysUserVo currentUser = ZHSWCommonUtils.getCurrentUser();
         for (SwWaterUsageEntryDo swWaterUsageEntry : collect) {
             String usernumber = swWaterUsageEntry.getUsernumber();
             // 本次抄表日期
@@ -145,10 +146,9 @@ public class BillGenerationServiceImpl {
                 saveUsageEntryList.add(usageEntryDo);
                 return;
             }
-
             try {
                 // 生成代缴费账单
-                swBillManagementUnpaidDo = produceBill(swUserManagementDo, swWaterUsageEntry);
+                swBillManagementUnpaidDo = produceBill(swUserManagementDo, swWaterUsageEntry, currentUser);
             } catch (Exception e) {
                 e.printStackTrace();
                 log.error("缴费记录生成失败原因:{}", e.getMessage());
@@ -168,6 +168,8 @@ public class BillGenerationServiceImpl {
             saveUsageEntryList.add(usageEntryDo);
         }
         dataSave(saveList, saveUsageEntryList, userSaveList);
+        long totalAsyncCost = System.currentTimeMillis() - asyncTasksStart;
+        log.info("账单生成记录耗时: {}ms", totalAsyncCost);
     }
 
 
@@ -236,7 +238,7 @@ public class BillGenerationServiceImpl {
             e.printStackTrace();
             transactionManager.rollback(status);
             log.error("账单保存异常");
-        }finally {
+        } finally {
             ContextHolder.clear();
         }
 
@@ -248,12 +250,12 @@ public class BillGenerationServiceImpl {
      *
      * @return
      */
-    public SwBillManagementUnpaidDo produceBill(SwUserManagementDo swUserManagementDo, SwWaterUsageEntryDo swWaterUsageEntryDo) {
+    public SwBillManagementUnpaidDo produceBill(SwUserManagementDo swUserManagementDo, SwWaterUsageEntryDo swWaterUsageEntryDo,
+                                                SysUserVo currentUser) {
         // 上次抄表数
         BigDecimal lastreading = swWaterUsageEntryDo.getLastreading() == null ? BigDecimal.ZERO : swWaterUsageEntryDo.getLastreading();
         // 本次抄表数
         BigDecimal thisreading = swWaterUsageEntryDo.getThisreading();
-
         //当前用水
         BigDecimal waterConsumption;
         // 本次大于上次
@@ -319,8 +321,8 @@ public class BillGenerationServiceImpl {
                 exemptionAmount = exemptionWater.multiply(waterprice);
             }
         }
-        log.debug("用户 {} 的减免计算：monthDiff={}, exemptionAmount={}, exemptionWater={}",
-                swUserManagementDo.getUsernumber(), monthDiff, exemptionAmount, exemptionWater);
+        // log.debug("用户 {} 的减免计算：monthDiff={}, exemptionAmount={}, exemptionWater={}",
+        //         swUserManagementDo.getUsernumber(), monthDiff, exemptionAmount, exemptionWater);
         swBillManagementUnpaidDo.setReducedwateruse(exemptionWater);
         swBillManagementUnpaidDo.setFeewaiver(exemptionAmount);
         swBillManagementUnpaidDo.setOughttohavepaid(waterprice.multiply(waterConsumption));
@@ -330,7 +332,7 @@ public class BillGenerationServiceImpl {
         swBillManagementUnpaidDo.setActualdue(subtract.compareTo(BigDecimal.ZERO) > 0 ? subtract : BigDecimal.ZERO);
 
         swBillManagementUnpaidDo.setStatus(1 + "");
-        ZHSWCommonUtils.initModelGeneralParameters(swBillManagementUnpaidDo, null);
+        ZHSWCommonUtils.initModelGeneralParameters(swBillManagementUnpaidDo, currentUser);
         swBillManagementUnpaidDo.setWaterusageid(swWaterUsageEntryDo.getId());
         return swBillManagementUnpaidDo;
     }

zhsw-common/src/main/java/com/rongwei/zhsw/system/service/impl/ImportExcelServiceImpl.java

@@ -1,6 +1,7 @@
 package com.rongwei.zhsw.system.service.impl;
 
 import com.alibaba.excel.EasyExcel;
+import com.google.common.collect.Lists;
 import com.rongwe.zhsw.system.domain.SwWaterUsageEntryDo;
 import com.rongwe.zhsw.system.vo.ImportMeterReadingRecordVo;
 import com.rongwei.rwadmincommon.system.vo.SysUserVo;
@@ -12,18 +13,20 @@ import com.rongwei.zhsw.system.importListener.MeterReadingRecordListener;
 import com.rongwei.zhsw.system.service.ImportExcelService;
 import com.rongwei.zhsw.system.utils.ZHSWCommonUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.ibatis.session.ExecutorType;
+import org.apache.ibatis.session.SqlSession;
+import org.apache.ibatis.session.SqlSessionFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Propagation;
-import org.springframework.transaction.annotation.Transactional;
-import org.springframework.web.context.request.RequestAttributes;
-import org.springframework.web.context.request.RequestContextHolder;
+import org.springframework.transaction.support.TransactionTemplate;
 
-import javax.servlet.http.HttpServletRequest;
 import java.io.File;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
 
 import static com.rongwei.rwcommon.utils.UtilsChecks.parameterCheck;
 
@@ -43,6 +46,14 @@ public class ImportExcelServiceImpl implements ImportExcelService {
     private SwWaterUsageEntryServiceImpl waterUsageEntryService;
     @Autowired
     private BillGenerationServiceImpl billGenerationService;
+    @Autowired
+    private SqlSessionFactory sqlSessionFactory;
+
+
+    @Qualifier("zhswThreadPool")
+    @Autowired
+    private Executor asyncTaskExecutor;
+
     private static final Logger log = LoggerFactory.getLogger(ImportExcelServiceImpl.class);
 
     /**
@@ -62,27 +73,50 @@ public class ImportExcelServiceImpl implements ImportExcelService {
         // 获取处理之后的数据
         long entTime = System.currentTimeMillis();
         log.error("excel解析时长：{}", startTime - entTime);
-        startTime = System.currentTimeMillis();
         List<SwWaterUsageEntryDo> saveList = meterReadingRecordListener.getData();
-        this.save(saveList);
-        entTime = System.currentTimeMillis();
-        log.error("数据保存时长：{}", startTime - entTime);
-        startTime = System.currentTimeMillis();
         SysUserVo currentUser = ZHSWCommonUtils.getCurrentUser();
-        ContextHolder.setValue("dsKey",currentUser.getTenantDo().getDskey());
-        billGenerationService.asyncGenerateBill(saveList);
-        entTime = System.currentTimeMillis();
-        log.error("账单生成时长：{}", startTime - entTime);
-        ContextHolder.clear();
+        String dskey = currentUser.getTenantDo().getDskey();
+        ContextHolder.setValue("dsKey", dskey);
+        this.dataSaveAndPostProcessing(saveList, dskey);
+
         return R.ok();
     }
 
-    @Transactional(propagation = Propagation.REQUIRES_NEW)
-    public void save(List<SwWaterUsageEntryDo> datas) {
-        waterUsageEntryService.saveBatch(datas, 1000);
-    }
+    // @Transactional(propagation = Propagation.REQUIRES_NEW)
+    public void dataSaveAndPostProcessing(List<SwWaterUsageEntryDo> datas, String dskey) {
+        long asyncTasksStart = System.currentTimeMillis();
+        List<List<SwWaterUsageEntryDo>> partition = Lists.partition(datas, 1000); // 动态分片
+        CompletableFuture<Void> allFutures = CompletableFuture.allOf(
+                partition.stream()
+                        .map(data ->
+                                CompletableFuture.runAsync(() -> {
+                                            long startTime = System.currentTimeMillis();
+                                            waterUsageEntryService.getBaseMapper().ListBatchSave(data, dskey);
+                                            log.debug("数据保存耗时: {}ms,总条数:{}", System.currentTimeMillis() - startTime, data.size());
+                                        },
+                                        asyncTaskExecutor).whenComplete((result, ex) -> {
+                                }))
+                        .toArray(CompletableFuture[]::new)
+        );
+
+        allFutures.thenRun(() -> {
+            log.debug("数据保存总耗时: {}ms", System.currentTimeMillis() - asyncTasksStart);
+            billGenerationService.asyncGenerateBill(datas); // 同步生成账单
+            ContextHolder.clear();
+        });
+        // partition.parallelStream().forEach(data -> {
+        //     waterUsageEntryService.getBaseMapper().ListBatchSave(data, dskey);
+        // });
+
+        // CompletableFuture<?>[] futures = partition.parallelStream()
+        //         .map(data -> CompletableFuture.runAsync(() ->
+        //                 waterUsageEntryService.getBaseMapper().ListBatchSave(data, dskey), asyncTaskExecutor)
+        //         )
+        //         .toArray(CompletableFuture[]::new);
 
 
+    }
+
     public File readFile(String fileId) {
         parameterCheck(() -> StringUtils.isBlank(fileId), "请上传文件", "文件ID为空");
         SysFileItemDo fileItemDo = sysFileItemDao.selectById(fileId);

zhsw-common/src/main/resources/mybatis/zhsw/SwWaterUsageEntryDao.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper
+        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
+        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.rongwei.zhsw.system.dao.SwWaterUsageEntryDao">
+
+    <resultMap id="BaseResultMap" type="com.rongwe.zhsw.system.domain.SwWaterUsageEntryDo">
+            <id property="id" column="ID" />
+            <result property="tenantid" column="TENANTID" />
+            <result property="roption" column="ROPTION" />
+            <result property="deleted" column="DELETED" />
+            <result property="remark" column="REMARK" />
+            <result property="createdate" column="CREATEDATE" />
+            <result property="createuserid" column="CREATEUSERID" />
+            <result property="modifydate" column="MODIFYDATE" />
+            <result property="modifyuserid" column="MODIFYUSERID" />
+            <result property="createusername" column="CREATEUSERNAME" />
+            <result property="modifyusername" column="MODIFYUSERNAME" />
+            <result property="userid" column="USERID" />
+            <result property="usernumber" column="USERNUMBER" />
+            <result property="username" column="USERNAME" />
+            <result property="thisreading" column="THISREADING" />
+            <result property="currentreadingdate" column="CURRENTREADINGDATE" />
+            <result property="lastreading" column="LASTREADING" />
+            <result property="lastreadingdate" column="LASTREADINGDATE" />
+            <result property="communitycode" column="COMMUNITYCODE" />
+            <result property="communityname" column="COMMUNITYNAME" />
+            <result property="address" column="ADDRESS" />
+            <result property="readingsource" column="READINGSOURCE" />
+            <result property="file" column="FILE" />
+            <result property="year" column="YEAR" />
+            <result property="month" column="MONTH" />
+            <result property="isds" column="ISDS" />
+            <result property="state" column="STATE" />
+    </resultMap>
+
+    <sql id="Base_Column_List">
+        ID,TENANTID,ROPTION,DELETED,REMARK,CREATEDATE,
+        CREATEUSERID,MODIFYDATE,MODIFYUSERID,CREATEUSERNAME,MODIFYUSERNAME,
+        USERID,USERNUMBER,USERNAME,THISREADING,CURRENTREADINGDATE,
+        LASTREADING,LASTREADINGDATE,COMMUNITYCODE,COMMUNITYNAME,ADDRESS,
+        READINGSOURCE,FILE,YEAR,MONTH,ISDS,
+        STATE
+    </sql>
+<!--    解决平台拦截器导致导入慢的问题 故使用select-->
+    <select id="ListBatchSave">
+        INSERT INTO ${dsKey}.sw_water_usage_entry (
+        ID,TENANTID,ROPTION,DELETED,REMARK,CREATEDATE,
+        CREATEUSERID,MODIFYDATE,MODIFYUSERID,CREATEUSERNAME,MODIFYUSERNAME,
+        USERID,USERNUMBER,USERNAME,THISREADING,CURRENTREADINGDATE,
+        LASTREADING,LASTREADINGDATE,COMMUNITYCODE,COMMUNITYNAME,ADDRESS,
+        READINGSOURCE,FILE,YEAR,MONTH,ISDS,
+        STATE
+        ) VALUES
+        <foreach collection="swWaterUsageEntryDos" item="item" separator=",">
+            (
+            #{item.id},
+            #{item.tenantid},
+            #{item.roption},
+            #{item.deleted},
+            #{item.remark},
+            #{item.createdate},
+            #{item.createuserid},
+            #{item.modifydate},
+            #{item.modifyuserid},
+            #{item.createusername},
+            #{item.modifyusername},
+            #{item.userid},
+            #{item.usernumber},
+            #{item.username},
+            #{item.thisreading},
+            #{item.currentreadingdate},
+            #{item.lastreading},
+            #{item.lastreadingdate},
+            #{item.communitycode},
+            #{item.communityname},
+            #{item.address},
+            #{item.readingsource},
+            #{item.file},
+            #{item.year},
+            #{item.month},
+            #{item.isds},
+            #{item.state}
+            )
+        </foreach>;
+    </select>
+</mapper>

zhsw-server/src/main/java/com/rongwei/ZHSWApplication.java

@@ -5,6 +5,7 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
 import org.springframework.cloud.openfeign.EnableFeignClients;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.annotation.EnableScheduling;
 
@@ -14,6 +15,7 @@ import org.springframework.scheduling.annotation.EnableScheduling;
 @MapperScan("com.rongwei.*.*.dao")
 @EnableAsync
 @EnableScheduling
+@EnableAspectJAutoProxy
 public class ZHSWApplication {
 
     public static void main(String[] args) {

zhsw-server/src/main/java/com/rongwei/zhsw/system/controller/OwnerController.java

@@ -2,6 +2,7 @@ package com.rongwei.zhsw.system.controller;
 
 import com.rongwei.rwcommon.base.R;
 import com.rongwei.zhsw.system.wechat.OwnerService;
+import com.rongwei.zhsw.system.wechat.impl.OwnerServiceImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,7 +26,7 @@ public class OwnerController {
     private final Logger log = LoggerFactory.getLogger(this.getClass().getName());
 
     @Autowired
-    private OwnerService ownerService;
+    private OwnerServiceImpl ownerService;
 
 
     /**

zhsw-server/src/main/resources/logback-spring.xml

@@ -25,7 +25,7 @@
     </appender>
 
     <!-- 指定包路径日志配置 -->
-    <logger name="com.rongwei" level="INFO" additivity="false">
+    <logger name="com.rongwei" level="DEBUG" additivity="false">
         <appender-ref ref="console"/>
         <appender-ref ref="logFile" />
     </logger>