Merge remote-tracking branch 'origin/master'

1 year ago · 55ce4c7e12
3 changed files with 118 additions and 100 deletions
--- a/src/main/java/com/biutag/supervisiondata/event/ApplicationEvent.java
+++ b/src/main/java/com/biutag/supervisiondata/event/ApplicationEvent.java
@ -158,7 +158,7 @@ public class ApplicationEvent {
 //            pointService.syncFamilySeriousIllness(initDomain(List.of(127)));
 //            pointService.syncPetitionDissatisfied(initDomain(List.of(128)));
 //            pointService.syncIsSpecialPeople(initDomain(List.of(129)));
-//            pointService.syncSpecialPeopleType(initDomain(List.of(130)));
+            pointService.syncSpecialPeopleType(initDomain(List.of(130)));
 //            pointService.syncParentsRelationship(initDomain(List.of(131)));
 //            pointService.syncMentalDiseaseDriver(initDomain(List.of(132)));
 //            pointService.syncXIDUDriver(initDomain(List.of(133)));
--- a/src/main/java/com/biutag/supervisiondata/mapper/dwd/GBaseCSCZRKMapper.java
+++ b/src/main/java/com/biutag/supervisiondata/mapper/dwd/GBaseCSCZRKMapper.java
@ -60,31 +60,31 @@ public interface GBaseCSCZRKMapper extends BaseMapper<GBaseCSCZRK> {

    @Select("select distinct a.gmsfhm, c.gmsfhm family_id from dwd_ry_kxgxzyk_csczrk a inner join csga_wdpcdb.wdpc_person b on a.gmsfhm = b.id_code "
            + "inner join dwd_ry_kxgxzyk_csczrk c ON a.hh = c.hh "
-            + "where a.gmsfhm != c.gmsfhm and ("
+            + "where a.gmsfhm != c.gmsfhm and "
            + "( a.mqgmsfzhm IS NULL AND a.fqgmsfzhm IS NULL ) "
-            + ")")
-    List<GBaseCSCZRK> selectFamilyPeople1();
+            + "LIMIT #{limit} OFFSET #{offset}")
+    List<GBaseCSCZRK> selectFamilyPeople1(int offset, int limit);

    @Select("select distinct a.gmsfhm, c.gmsfhm family_id from dwd_ry_kxgxzyk_csczrk a inner join csga_wdpcdb.wdpc_person b on a.gmsfhm = b.id_code "
            + "inner join dwd_ry_kxgxzyk_csczrk c ON a.hh = c.hh "
-            + "where a.gmsfhm != c.gmsfhm and ("
+            + "where a.gmsfhm != c.gmsfhm and "
            + "( a.mqgmsfzhm IS NULL AND a.fqgmsfzhm != c.gmsfhm ) "
-            + ")")
-    List<GBaseCSCZRK> selectFamilyPeople2();
+            + "LIMIT #{limit} OFFSET #{offset}")
+    List<GBaseCSCZRK> selectFamilyPeople2(int offset, int limit);

    @Select("select distinct a.gmsfhm, c.gmsfhm family_id from dwd_ry_kxgxzyk_csczrk a inner join csga_wdpcdb.wdpc_person b on a.gmsfhm = b.id_code "
            + "inner join dwd_ry_kxgxzyk_csczrk c ON a.hh = c.hh "
-            + "where a.gmsfhm != c.gmsfhm and ("
+            + "where a.gmsfhm != c.gmsfhm and "
            + "( a.fqgmsfzhm IS NULL AND a.mqgmsfzhm != c.gmsfhm ) "
-            + ")")
-    List<GBaseCSCZRK> selectFamilyPeople3();
+            + "LIMIT #{limit} OFFSET #{offset}")
+    List<GBaseCSCZRK> selectFamilyPeople3(int offset, int limit);

    @Select("select distinct a.gmsfhm, c.gmsfhm family_id from dwd_ry_kxgxzyk_csczrk a inner join csga_wdpcdb.wdpc_person b on a.gmsfhm = b.id_code "
            + "inner join dwd_ry_kxgxzyk_csczrk c ON a.hh = c.hh "
-            + "where a.gmsfhm != c.gmsfhm and ("
+            + "where a.gmsfhm != c.gmsfhm and "
            + "(a.fqgmsfzhm IS NOT NULL AND a.mqgmsfzhm IS NOT NULL AND a.fqgmsfzhm != c.gmsfhm AND a.mqgmsfzhm != c.gmsfhm) "
-            + ")")
-    List<GBaseCSCZRK> selectFamilyPeople4();
+            + "LIMIT #{limit} OFFSET #{offset}")
+    List<GBaseCSCZRK> selectFamilyPeople4(int offset, int limit);

    @Select("select distinct a.gmsfhm, a.fqgmsfzhm father_id from dwd_ry_kxgxzyk_csczrk a INNER JOIN csga_wdpcdb.wdpc_person b ON a.gmsfhm = b.id_code where a.fqgmsfzhm is not null;")
    List<GBaseCSCZRK> selectFather();
--- a/src/main/java/com/biutag/supervisiondata/service/impl/PointServiceImpl.java
+++ b/src/main/java/com/biutag/supervisiondata/service/impl/PointServiceImpl.java
@ -27,6 +27,7 @@ import com.biutag.supervisiondata.util.IdCodeUtil;
 import com.biutag.supervisiondata.util.PatternUtil;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Service;

 import java.text.SimpleDateFormat;
@ -832,36 +833,15 @@ public class PointServiceImpl implements PointService {
        Map<String, RiskModelTaskClue> oldMap = domain.getOldMap(task.getModelId(), true);
        Map<String, Integer> idCode = new HashMap<>();
        log.info("模型130开始查询");
-        List<WdpcGrjdJsbry> list1 = jsbryRepository.getBaseMapper().selectByIdNos();
+//        List<WdpcGrjdJsbry> list1 = jsbryRepository.getBaseMapper().selectByIdNos();
        List<WdpcJZJLB> list2 = jZJLBRepository.getBaseMapper().selectForModel129();
        ;
        List<WdpcXdry> list3 = xdryRepository.getBaseMapper().selectXIDU();
        List<WdpcSqjzry> list4 = sqjzryRepository.getBaseMapper().selectByIdNos();
-        List<WdpcXmsfry> list5 = xmsfryRepository.getBaseMapper().selectByIdNos();
-        log.info("模型130查询结果数：{},{},{},{},{}", list1.size(), list2.size(), list3.size(), list4.size(), list5.size());
+//        List<WdpcXmsfry> list5 = xmsfryRepository.getBaseMapper().selectByIdNos();
+        log.info("模型130查询结果数：{},{},{}", list2.size(), list3.size(), list4.size());
        Map<String, RiskPersonal> personalMap = domain.getPersons().stream().collect(Collectors.toMap(RiskPersonal::getIdCode, Function.identity(), (old, newVal) -> newVal));
        List<RiskModelTaskClue> toInsert = new ArrayList<>();
-        for (WdpcGrjdJsbry domicileType : list1) {
-            RiskPersonal personal = personalMap.get(domicileType.getIdCode());
-            if (personal == null) continue;
-            if (idCode.get(domicileType.getIdCode()) != null) continue;
-            if (oldMap.get(domicileType.getIdCode()) != null) continue;
-
-            idCode.put(domicileType.getIdCode(), 1);
-
-            ClueData data = ClueData.builder()
-                    .name(personal.getName())
-                    .id(personal.getId().toString())
-                    .idCode(personal.getIdCode())
-                    .tags(List.of("精神障碍患者"))
-                    .score(1)
-                    .eventTime(LocalDateTime.now())
-                    .personId(personal.getId())
-                    .sourceData("发现该人员于" + getDateString(domicileType.getActualTime())
-                            + "确诊为为“" + domicileType.getActualId() + "”精神障碍患者")
-                    .build();
-            toInsert.add(DataUtil.createClueData(task, data));
-        }
        for (WdpcJZJLB domicileType : list2) {
            RiskPersonal personal = personalMap.get(domicileType.getIdCode());
            if (personal == null) continue;
@ -938,42 +918,6 @@ public class PointServiceImpl implements PointService {
                    .build();
            toInsert.add(DataUtil.createClueData(task, data));
        }
-        for (WdpcXmsfry domicileType : list5) {
-            RiskPersonal personal = personalMap.get(domicileType.getIdCode());
-            if (personal == null) continue;
-            if (idCode.get(domicileType.getIdCode()) != null) continue;
-            if (oldMap.get(domicileType.getIdCode()) != null) continue;
-
-            idCode.put(domicileType.getIdCode(), 1);
-
-            String context = "";
-            if (domicileType.getCrimeName() != null) {
-                context = context + "，罪名为" + domicileType.getCrimeName();
-            }
-            if (domicileType.getSentenceTime() != null) {
-                context = context + "，判决日期为" + getDateString(domicileType.getSentenceTime());
-            }
-            if (domicileType.getPrisonBegin() != null) {
-                context = context + "，服刑开始日期为" + getDateString(domicileType.getPrisonBegin());
-            }
-            if (domicileType.getPrisonEnd() != null) {
-                context = context + "，服刑结束日期为" + getDateString(domicileType.getPrisonEnd());
-            }
-            if (domicileType.getPrisonOrganization() != null) {
-                context = context + "，服刑单位为" + domicileType.getPrisonOrganization();
-            }
-            ClueData data = ClueData.builder()
-                    .name(personal.getName())
-                    .id(personal.getId().toString())
-                    .idCode(personal.getIdCode())
-                    .tags(List.of("刑释解教人员"))
-                    .score(5)
-                    .eventTime(LocalDateTime.now())
-                    .personId(personal.getId())
-                    .sourceData("发现该人员为刑释解教人员" + context)
-                    .build();
-            toInsert.add(DataUtil.createClueData(task, data));
-        }
        if (!toInsert.isEmpty()) {
            modelTaskClueService.saveClues(toInsert, Default.SPECIALPEOPLETYPE);
        }
@ -1454,27 +1398,107 @@ public class PointServiceImpl implements PointService {

        Map<String, RiskModelTaskClue> oldMap = domain.getOldMap(task.getModelId(), true);
        Map<String, Integer> idCode = new HashMap<>();
-        List<String> list = new ArrayList<>();
        log.info("模型118开始查询");
-        List<GBaseCSCZRK> familyPeople = new ArrayList<>(cSCZRKRepository.getBaseMapper().selectFamilyPeople1());
-        log.info("模型118查询familyPeople1结果数：{}", familyPeople.size());
-        familyPeople.addAll(cSCZRKRepository.getBaseMapper().selectFamilyPeople2());
-        log.info("模型118查询familyPeople2结果数：{}", familyPeople.size());
-        familyPeople.addAll(cSCZRKRepository.getBaseMapper().selectFamilyPeople3());
-        log.info("模型118查询familyPeople3结果数：{}", familyPeople.size());
-        familyPeople.addAll(cSCZRKRepository.getBaseMapper().selectFamilyPeople4());
-        log.info("模型118查询familyPeople结果数：{}", familyPeople.size());
+        int pageSize = 100000;
+        int pageNumber1 = 1;
+        List<GBaseCSCZRK> currentPageResults1;
+        do {
+            int offset = (pageNumber1 - 1) * pageSize;
+            currentPageResults1 = cSCZRKRepository.getBaseMapper().selectFamilyPeople1(offset, pageSize);
+            log.info("模型118第一部分查询size:{}", currentPageResults1.size());
+            if (currentPageResults1.isEmpty()) break;
+            List<GBaseCSCZRK> familyPeople = new ArrayList<>(currentPageResults1);
            Map<String, List<String>> familyMap = familyPeople.stream().collect(Collectors.groupingBy(GBaseCSCZRK::getGmsfhm,
                    Collectors.mapping(GBaseCSCZRK::getFamilyId, Collectors.toList())));
            List<String> familyList = familyPeople.stream().map(GBaseCSCZRK::getFamilyId).toList();
-        // 切割 每次查10000条
-        List<List<String>> familyListParts = IntStream.range(0, (familyList.size() + 9999) / 10000)
-                .mapToObj(i -> familyList.subList(i * 10000, Math.min((i + 1) * 10000, familyList.size())))
-                .toList();
-        for (List<String> familyListPart : familyListParts) {
-            list.addAll(hZCRYXXRepository.getBaseMapper().selectHZCPeople(familyListPart));
+            List<String> list = new ArrayList<>(hZCRYXXRepository.getBaseMapper().selectHZCPeople(familyList));
+            log.info("模型118第一部分查询list:{}", list.size());
+            List<RiskModelTaskClue> toInsert = getRiskModelTaskClues(domain, familyMap, list, idCode, oldMap, task);
+            RiskTask updateTask = new RiskTask();
+            updateTask.setId(task.getId());
+            updateTask.setInsertSize(toInsert.size());
+            updateTask.setUpdateSize(0);
+            updateTask.setState(1);
+            updateTask.setEndTime(LocalDateTime.now());
+            taskRepository.updateById(updateTask);
+            pageNumber1++;
+        } while (!currentPageResults1.isEmpty());
+        log.info("模型118查询familyPeople1完成");
+        int pageNumber2 = 1;
+        List<GBaseCSCZRK> currentPageResults2;
+        do {
+            int offset = (pageNumber2 - 1) * pageSize;
+            currentPageResults2 = cSCZRKRepository.getBaseMapper().selectFamilyPeople2(offset, pageSize);
+            log.info("模型118第二部分查询size:{}", currentPageResults2.size());
+            if (currentPageResults2.isEmpty()) break;
+            List<GBaseCSCZRK> familyPeople = new ArrayList<>(currentPageResults2);
+            Map<String, List<String>> familyMap = familyPeople.stream().collect(Collectors.groupingBy(GBaseCSCZRK::getGmsfhm,
+                    Collectors.mapping(GBaseCSCZRK::getFamilyId, Collectors.toList())));
+            List<String> familyList = familyPeople.stream().map(GBaseCSCZRK::getFamilyId).toList();
+            List<String> list = new ArrayList<>(hZCRYXXRepository.getBaseMapper().selectHZCPeople(familyList));
+            log.info("模型118第二部分查询list:{}", list.size());
+            List<RiskModelTaskClue> toInsert = getRiskModelTaskClues(domain, familyMap, list, idCode, oldMap, task);
+            RiskTask updateTask = new RiskTask();
+            updateTask.setId(task.getId());
+            updateTask.setInsertSize(toInsert.size());
+            updateTask.setUpdateSize(0);
+            updateTask.setState(1);
+            updateTask.setEndTime(LocalDateTime.now());
+            taskRepository.updateById(updateTask);
+            pageNumber2++;
+        } while (!currentPageResults2.isEmpty());
+        log.info("模型118查询familyPeople2完成");
+        int pageNumber3 = 1;
+        List<GBaseCSCZRK> currentPageResults3;
+        do {
+            int offset = (pageNumber3 - 1) * pageSize;
+            currentPageResults3 = cSCZRKRepository.getBaseMapper().selectFamilyPeople3(offset, pageSize);
+            log.info("模型118第三部分查询size:{}", currentPageResults3.size());
+            if (currentPageResults3.isEmpty()) break;
+            List<GBaseCSCZRK> familyPeople = new ArrayList<>(currentPageResults3);
+            Map<String, List<String>> familyMap = familyPeople.stream().collect(Collectors.groupingBy(GBaseCSCZRK::getGmsfhm,
+                    Collectors.mapping(GBaseCSCZRK::getFamilyId, Collectors.toList())));
+            List<String> familyList = familyPeople.stream().map(GBaseCSCZRK::getFamilyId).toList();
+            List<String> list = new ArrayList<>(hZCRYXXRepository.getBaseMapper().selectHZCPeople(familyList));
+            log.info("模型118第三部分查询list:{}", list.size());
+            List<RiskModelTaskClue> toInsert = getRiskModelTaskClues(domain, familyMap, list, idCode, oldMap, task);
+            RiskTask updateTask = new RiskTask();
+            updateTask.setId(task.getId());
+            updateTask.setInsertSize(toInsert.size());
+            updateTask.setUpdateSize(0);
+            updateTask.setState(1);
+            updateTask.setEndTime(LocalDateTime.now());
+            taskRepository.updateById(updateTask);
+            pageNumber3++;
+        } while (!currentPageResults3.isEmpty());
+        log.info("模型118查询familyPeople3完成");
+        int pageNumber4 = 1;
+        List<GBaseCSCZRK> currentPageResults4;
+        do {
+            int offset = (pageNumber4 - 1) * pageSize;
+            currentPageResults4 = cSCZRKRepository.getBaseMapper().selectFamilyPeople3(offset, pageSize);
+            log.info("模型118第四部分查询size:{}", currentPageResults4.size());
+            if (currentPageResults4.isEmpty()) break;
+            List<GBaseCSCZRK> familyPeople = new ArrayList<>(currentPageResults4);
+            Map<String, List<String>> familyMap = familyPeople.stream().collect(Collectors.groupingBy(GBaseCSCZRK::getGmsfhm,
+                    Collectors.mapping(GBaseCSCZRK::getFamilyId, Collectors.toList())));
+            List<String> familyList = familyPeople.stream().map(GBaseCSCZRK::getFamilyId).toList();
+            List<String> list = new ArrayList<>(hZCRYXXRepository.getBaseMapper().selectHZCPeople(familyList));
+            log.info("模型118第四部分查询list:{}", list.size());
+            List<RiskModelTaskClue> toInsert = getRiskModelTaskClues(domain, familyMap, list, idCode, oldMap, task);
+            RiskTask updateTask = new RiskTask();
+            updateTask.setId(task.getId());
+            updateTask.setInsertSize(toInsert.size());
+            updateTask.setUpdateSize(0);
+            updateTask.setState(1);
+            updateTask.setEndTime(LocalDateTime.now());
+            taskRepository.updateById(updateTask);
+            pageNumber4++;
+        } while (!currentPageResults4.isEmpty());
+        log.info("模型118查询familyPeople4完成");
    }
-        log.info("模型118查询结果数：{}", list.size());
+
+    private @NotNull List<RiskModelTaskClue> getRiskModelTaskClues(TaskParamDomain domain, Map<String, List<String>> familyMap, List<String> list, Map<String, Integer> idCode, Map<String, RiskModelTaskClue> oldMap, RiskTask task) {
        Map<String, RiskPersonal> personalMap = domain.getPersons().stream().collect(Collectors.toMap(RiskPersonal::getIdCode, Function.identity(), (old, newVal) -> newVal));
        List<RiskModelTaskClue> toInsert = new ArrayList<>();
        for (Map.Entry<String, List<String>> entry : familyMap.entrySet()) {
@ -1507,13 +1531,7 @@ public class PointServiceImpl implements PointService {
        if (!toInsert.isEmpty()) {
            modelTaskClueService.saveClues(toInsert, Default.FAMILYDIE);
        }
-        RiskTask updateTask = new RiskTask();
-        updateTask.setId(task.getId());
-        updateTask.setInsertSize(toInsert.size());
-        updateTask.setUpdateSize(0);
-        updateTask.setState(1);
-        updateTask.setEndTime(LocalDateTime.now());
-        taskRepository.updateById(updateTask);
+        return toInsert;
    }

    @Override