优化学历和地区的查询速度

src/main/java/com/biutag/supervisiondata/event/ApplicationEvent.java

@@ -114,7 +114,7 @@ public class ApplicationEvent {
 //            personalService.syncPerson();
 //
 //            petitionService.petitionModelTask(initDomain(List.of(40)));
-//            riskTaskService.eduModelTask(initDomain(List.of(34)));
+//            riskTaskService.eduModelTask(initDomain(List.of(34,41)));
 //            riskTaskService.infoModelTask(initDomain(List.of(32,33)));
 //            marriageService.marriageModelTask(initDomain(List.of(39,51)));
 //            pointService.syncSecurity(initDomain(List.of(37)));

src/main/java/com/biutag/supervisiondata/mapper/ads/GBaseSYRKXXMapper.java

@@ -63,7 +63,11 @@ public interface GBaseSYRKXXMapper extends BaseMapper<GBaseSYRKXX> {
             "</script>")
     List<FightMan> selectListByIdCode(@Param("list")List<String> list);
 
-    @Select("select distinct sspcsdm from ads_ssxx_syrkxx where sspcsdm is not null")
-    List<GBaseSYRKXX> selectOrg();
+    /**
+     * 风险人员的学历和职业
+     * @return
+     */
+    @Select(" select a.gmsfhm as gmsfhm, a.xlmc as xlmc, a.csdxzqhdm as csdxzqhdm, a.csdxzqhmc as csdxzqhmc from csga_ads.ads_ssxx_syrkxx as a inner join csga_wdpcdb.wdpc_person as b on a.gmsfhm = b.id_code ")
+    List<GBaseSYRKXX> selectEduAndArea();
 
 }

src/main/java/com/biutag/supervisiondata/mapper/wdpc/WdpcPersonMapper.java

@@ -27,4 +27,5 @@ public interface WdpcPersonMapper extends BaseMapper<WdpcPerson> {
     @Select(" select a.GMSFHM as idCode,a.sdxp as image  from csga_dwd.dwd_ry_qbpt_sfzzzxx as a inner join csga_wdpcdb.wdpc_person as b " +
             "on a.GMSFHM = b.id_code limit #{pageNo}, #{pageSize}")
     List<PersonImageInfo> selectPersonImagePage(@Param("pageNo")Integer pageNo, @Param("pageSize")Integer pageSize);
+
 }

src/main/java/com/biutag/supervisiondata/repository/WdpcPersonRepository.java

@@ -13,7 +13,4 @@ import org.springframework.stereotype.Service;
 @Service
 public class WdpcPersonRepository extends ServiceImpl<WdpcPersonMapper, WdpcPerson> {
 
-    void selectAndSave(int pageNo, int pageSize) {
-
-    }
 }

src/main/java/com/biutag/supervisiondata/service/RiskTaskService.java

@@ -20,6 +20,8 @@ public interface RiskTaskService {
      */
     void eduModelTask(TaskParamDomain domain);
 
+    void eduAndAreaModelTask(TaskParamDomain domain);
+
     /**
      * （三）行为因素
      * @param domain 参数

src/main/java/com/biutag/supervisiondata/service/impl/MarriageServiceImpl.java

@@ -58,7 +58,7 @@ public class MarriageServiceImpl implements MarriageService {
         RiskTask task2 = riskTaskRepository.create(data.size(), 51);
 
         Map<String, WdpcGrjdHyqkb> dataMap = data.stream().collect(Collectors.toMap(WdpcGrjdHyqkb::getIdCode, Function.identity(), (val, old) -> val));
-        log.info("开始查询结婚数据：{}", domain.getIdCards().size());
+        log.info("开始查询结婚数据：{}", dataMap.size());
 
         Map<String, RiskModelTaskClue> old1 = domain.getOldMap(39, true);
         Map<String, RiskModelTaskClue> old2 = domain.getOldMap(51, true);

src/main/java/com/biutag/supervisiondata/service/impl/PersonalServiceImpl.java

@@ -519,6 +519,7 @@ public class PersonalServiceImpl implements PersonalService {
         log.info("有效头像共：{}", total);
         List<PersonImageInfo> infoList;
         List<RiskPersonInfo> infos;
+        Map<String, Byte[]> p = new WeakHashMap<>();
         for (int page = 1;; page++) {
             log.info("{}", page);
             int pageNo = (page- 1) * 10000;
@@ -527,12 +528,16 @@ public class PersonalServiceImpl implements PersonalService {
             infos = new ArrayList<>();
             log.info("图像转base64开始：{}",infoList.size());
             for (PersonImageInfo item : infoList) {
+                if(p.get(item.getIdCode().toUpperCase()) != null) {
+                    continue;
+                }
+                p.put(item.getIdCode().toUpperCase(), new Byte[0]);
                 RiskPersonInfo info = new RiskPersonInfo();
                 info.setIdCode(item.getIdCode());
                 info.setImageBase64(Media.JPEG.concat(Base64.getEncoder().encodeToString(item.getImage())));
                 infos.add(info);
             }
-            infoRepository.saveBatch(infos, 1000);
+            infoRepository.saveOrUpdateBatch(infos, 1000);
             if(pageNo + pageSize >= total) {
                 break;
             }

src/main/java/com/biutag/supervisiondata/service/impl/RiskTaskServiceImpl.java

@@ -56,22 +56,12 @@ public class RiskTaskServiceImpl implements RiskTaskService {
 
     private final GBaseSYRKXXRepository syrkxxRepository;
 
-    private final GBaseWFRXXRepository wfrxxRepository;
-
-    private final GBaseAJJBXXRepository ajjbxxRepository;
-
-    private final RiskPersonalRepository riskPersonalRepository;
-
-    private final RiskPersonalControlRecordService controlRecordRepository;
-
     private final RiskModelTaskClueRepository clueRepository;
 
     private final RiskModelTaskClueService riskModelTaskClueService;
 
     private final RiskPersonalService riskPersonalService;
 
-    private final RiskPersonalControlRecordService controlRecordService;
-
     @Override
     public void infoModelTask(TaskParamDomain domain) {
         RiskTask genderTask = riskTaskRepository.create(domain.getPersons().size(), 32);
@@ -116,8 +106,10 @@ public class RiskTaskServiceImpl implements RiskTaskService {
             String edu = switch (xx.getXlmc()) {
                 case "小学教育", "文盲", "小学毕业", "小学肄业", "其他" -> "小学及以下";
                 case "初中毕业", "初级中等教育", "初中肄业" -> "初中";
-                case "普通高中结业", "普通高级中学教育", "普通高中毕业", "普通高中肄业","中等专科结业", "中等专科教育", "中等专业毕业", "技工学校肄业" -> "高中/中专";
-                case "大学专科毕业", "大学专科结业", "大学本科/专科教育", "大学生本科结业", "大学生普通班毕业","大学生本科肄业","大学生本科毕业" -> "大学";
+                case "普通高中结业", "普通高级中学教育", "普通高中毕业", "普通高中肄业", "中等专科结业", "中等专科教育", "中等专业毕业", "技工学校肄业" ->
+                        "高中/中专";
+                case "大学专科毕业", "大学专科结业", "大学本科/专科教育", "大学生本科结业", "大学生普通班毕业", "大学生本科肄业", "大学生本科毕业" ->
+                        "大学";
                 case "硕士研究生毕业", "研究生教育", "研究生班肄业" -> "大学以上";
                 default -> null;
 
@@ -160,6 +152,78 @@ public class RiskTaskServiceImpl implements RiskTaskService {
         riskTaskRepository.updateTask(eduTask.getId(), toInsert.size(), toUpdate.size());
     }
 
+    @Override
+    public void eduAndAreaModelTask(TaskParamDomain domain) {
+        // 学历
+        RiskTask eduTask = riskTaskRepository.create(domain.getPersons().size(), 34);
+        // 区域
+        RiskTask areaTask = riskTaskRepository.create(domain.getPersons().size(), 41);
+
+        List<GBaseSYRKXX> xxList = syrkxxRepository.getBaseMapper().selectEduAndArea();
+        Map<String, RiskModelTaskClue> oldEduMap = domain.getOldMap(34, true);
+        Map<String, RiskModelTaskClue> oldAreaMap = domain.getOldMap(41, true);
+
+        Map<String, GBaseSYRKXX> xxMap = xxList.stream().collect(Collectors.toMap(GBaseSYRKXX::getGmsfhm, Function.identity(), (oldValue, newValue) -> newValue));
+        List<RiskModelTaskClue> toInsert1 = new ArrayList<>();
+        List<RiskModelTaskClue> toInsert2 = new ArrayList<>();
+
+        String areCode = "431122,431322,433127,431382,430525,430722";
+        for (RiskPersonal person : domain.getPersons()) {
+            GBaseSYRKXX xx = xxMap.get(person.getIdCode());
+            if (xx == null) {
+                continue;
+            }
+            if (xx.getXlmc() != null && oldEduMap.get(person.getIdCode()) == null) {
+                String edu = switch (xx.getXlmc()) {
+                    case "小学教育", "文盲", "小学毕业", "小学肄业", "其他" -> "小学及以下";
+                    case "初中毕业", "初级中等教育", "初中肄业" -> "初中";
+                    case "普通高中结业", "普通高级中学教育", "普通高中毕业", "普通高中肄业", "中等专科结业", "中等专科教育", "中等专业毕业", "技工学校肄业" ->
+                            "高中/中专";
+                    case "大学专科毕业", "大学专科结业", "大学本科/专科教育", "大学生本科结业", "大学生普通班毕业", "大学生本科肄业", "大学生本科毕业" ->
+                            "大学";
+                    case "硕士研究生毕业", "研究生教育", "研究生班肄业" -> "大学以上";
+                    default -> null;
+                };
+                if (edu != null) {
+                    Integer score = Edu.contains(edu);
+                    //创建新的
+                    toInsert1.add(createClue(eduTask.getModelId(), score, eduTask.getId(), ClueData.builder()
+                            .name(person.getName())
+                            .idCode(person.getIdCode())
+                            .tag(edu)
+                            .sourceData("该人员的学历为" + xx.getXlmc() + "，属于" + edu)
+                            .eventTime(LocalDateTime.now())
+                            .personId(person.getId())
+                            .build(), "edu" + person.getId().toString()));
+                }
+            }
+            if(xx.getCsdxzqhdm() != null && oldAreaMap.get(person.getIdCode()) == null) {
+                if(!areCode.contains(xx.getCsdxzqhdm())) {
+                    continue;
+                }
+                toInsert2.add(createClue(areaTask.getModelId(), 5, areaTask.getId(), ClueData.builder()
+                        .id(BhUtil.sfBh(person.getId().toString()))
+                        .name(person.getName())
+                        .idCode(person.getIdCode())
+                        .personId(person.getId())
+                        .eventTime(LocalDateTime.now())
+                        .sourceData("发现该人员在" + xx.getCsdxzqhmc() + "成长，该地区暴力警情情况较多")
+                        .tag(xx.getCsdxzqhmc())
+                        .build(), "area___" + person.getId().toString()));
+            }
+        }
+        if(!toInsert1.isEmpty()) {
+            log.info("保存学历：{}", toInsert1.size());
+            riskModelTaskClueService.saveClues(toInsert1, Default.EDU_IG);
+        }
+        riskTaskRepository.updateTask(eduTask.getId(), toInsert1.size(), 0);
+        if(!toInsert2.isEmpty()) {
+            log.info("保存风险区域：{}", toInsert2.size());
+            riskModelTaskClueService.saveClues(toInsert2, Default.FIGHT_IG);
+        }
+        riskTaskRepository.updateTask(areaTask.getId(), toInsert2.size(), 0);
+    }
+
     @Override
     public void behavior(TaskParamDomain domain) {
         RiskTask task = riskTaskRepository.create(domain.getPersons().size(), 20);