新增数据工坊模型导入功能

src/main/java/com/biutag/supervisiondata/mapper/mine/ModelGenerationMapper.java

@@ -0,0 +1,8 @@
+package com.biutag.supervisiondata.mapper.mine;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.biutag.supervisiondata.pojo.entity.mine.ModelGeneration;
+
+public interface ModelGenerationMapper extends BaseMapper<ModelGeneration> {
+
+}

src/main/java/com/biutag/supervisiondata/mapper/wdpc2/ModelClueWdpc2Mapper.java

@@ -0,0 +1,15 @@
+package com.biutag.supervisiondata.mapper.wdpc2;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.biutag.supervisiondata.pojo.entity.mine.ModelClue;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+
+import java.util.List;
+import java.util.Map;
+
+public interface ModelClueWdpc2Mapper extends BaseMapper<ModelClue> {
+
+    @Select("${newSql}")
+    List<Map<String, Object>> selectByUniqueKeys(@Param("newSql") String newSql);
+}

src/main/java/com/biutag/supervisiondata/pojo/entity/mine/Model.java

@@ -93,4 +93,54 @@ public class Model {
     private String modelSql;
 
     private Integer riskScoreRuleId;
+
+    // 通知单位
+    private String handleDepartType;
+
+    private String handleDepartId;
+
+    // 业务类型
+    private String businessTypeCode;
+
+    // 警种
+    private String policeType;
+
+    private String involveProblem;
+
+    // 模型类型
+    @TableField("model_Type")
+    private String modelType;
+
+    // 是否要求回复
+    private Boolean requestReply;
+
+    // 通知回复时限
+    private Integer replyLimit;
+
+    //-------------------------------
+
+    // 模型结果表名
+    private String clueTableName;
+
+    // 同步时间字段名
+    private String clueTimeFieldName;
+
+    // 同步唯一字段名
+    private String clueUniqueFieldName;
+
+    // 同步周期 day-每天 week-每周
+    private String clueCycle;
+
+    // 同步周期 周 MON TUE WED THU FRI SAT SUN
+    private String clueCycleDayOfWeek;
+
+    // 同步时间
+    private String clueCycleTime;
+
+    // 同步机构映射（情指行，执法办案等）
+    private String clueDepartSource;
+
+    // 最近运行时间
+    @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss")
+    private LocalDateTime latestRunTime;
 }

src/main/java/com/biutag/supervisiondata/pojo/entity/mine/ModelClue.java

@@ -6,12 +6,14 @@ import com.baomidou.mybatisplus.annotation.TableId;
 import com.fasterxml.jackson.annotation.JsonFormat;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.experimental.Accessors;
 
 import java.math.BigDecimal;
 import java.time.LocalDateTime;
 
 @Setter
 @Getter
+@Accessors(chain = true)
 public class ModelClue {
 
     //

src/main/java/com/biutag/supervisiondata/pojo/entity/mine/ModelGeneration.java

@@ -0,0 +1,55 @@
+package com.biutag.supervisiondata.pojo.entity.mine;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.LocalDateTime;
+
+@Setter
+@Getter
+public class ModelGeneration {
+
+    //
+    @TableId(value = "id", type = IdType.AUTO)
+    private Integer id;
+
+    //
+    @TableField("model_id")
+    private Integer modelId;
+
+    // 涉及单位
+    @TableField("involve_depart_name")
+    private String involveDepartName;
+
+    // 涉及单位ID
+    @TableField("involve_depart_id")
+    private String involveDepartId;
+
+    // 涉及人员
+    @TableField("involve_police_name")
+    private String involvePoliceName;
+
+    // 涉及人员警号
+    @TableField("involve_police_emp_no")
+    private String involvePoliceEmpNo;
+
+    // 发生时间
+    @TableField("happen_time")
+    private String happenTime;
+
+    // 预警内容生成器
+    @TableField("thing_desc_generation")
+    private String thingDescGeneration;
+
+    // 创建时间
+    @TableField("create_time")
+    private LocalDateTime createTime;
+
+    // 更新时间
+    @TableField("update_time")
+    private LocalDateTime updateTime;
+
+}

src/main/java/com/biutag/supervisiondata/repository/ModelGenerationRepository.java

@@ -0,0 +1,16 @@
+package com.biutag.supervisiondata.repository;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.biutag.supervisiondata.mapper.mine.ModelGenerationMapper;
+import com.biutag.supervisiondata.pojo.entity.mine.ModelGeneration;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author kami on 2024-11-18 19:50:06
+ * @version 0.0.1
+ * @since 1.8
+ */
+@Service
+public class ModelGenerationRepository extends ServiceImpl<ModelGenerationMapper, ModelGeneration> {
+}

src/main/java/com/biutag/supervisiondata/rest/ApiController.java

@@ -22,8 +22,6 @@ import org.springframework.web.multipart.MultipartFile;
 
 import java.io.File;
 import java.io.IOException;
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
 import java.util.*;
 
 /**
@@ -160,6 +158,12 @@ public class ApiController {
         return RS.success();
     }
 
+    @GetMapping("/model/custom")
+    public RS<Void> scoreCustom() {
+        modelClueService.generateCustomModels();
+        return RS.success();
+    }
+
     /**
      * 图片上传
      * @param file 图片文件

src/main/java/com/biutag/supervisiondata/service/ModelClueService.java

@@ -2,4 +2,6 @@ package com.biutag.supervisiondata.service;
 
 public interface ModelClueService {
     void generate(Integer modelId);
+
+    void generateCustomModels();
 }

src/main/java/com/biutag/supervisiondata/service/impl/ModelClueServiceImpl.java

@@ -9,11 +9,13 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.biutag.supervisiondata.mapper.dwd.ModelClueDataMapper;
 import com.biutag.supervisiondata.mapper.mine.ModelClueMapper;
+import com.biutag.supervisiondata.mapper.wdpc2.ModelClueWdpc2Mapper;
 import com.biutag.supervisiondata.pojo.entity.mine.*;
 import com.biutag.supervisiondata.repository.*;
 import com.biutag.supervisiondata.service.ModelClueService;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Service;
 
 import java.math.BigDecimal;
@@ -39,10 +41,14 @@ public class ModelClueServiceImpl implements ModelClueService {
 
     private final ModelClueRepository modelClueRepository;
 
+    private final ModelGenerationRepository modelGenerationRepository;
+
     public final ModelClueMapper modelClueMapper;
 
     public final ModelClueDataMapper modelClueDataMapper;
 
+    public final ModelClueWdpc2Mapper modelClueWdpc2Mapper;
+
     public final ModelClueRecordRepository modelClueRecordRepository;
 
     public final SupExternalDepartRepository supExternalDepartRepository;
@@ -60,10 +66,12 @@ public class ModelClueServiceImpl implements ModelClueService {
         }
         Model model = modelRepository.getById(modelId);
         List<ModelClue> modelClues = new ArrayList<>();
-        if (!Objects.isNull(model) && model.getModelSql() != null) {
-            modelClues.addAll(getModelClueBySql(model.getModelSql(), modelId));
+        if (!Objects.isNull(model) && model.getModelingMethod().equals("2")) {
+            modelClues.addAll(getModelClueBySql("customModel", modelId, model));
+        } else if (!Objects.isNull(model) && model.getModelSql() != null) {
+            modelClues.addAll(getModelClueBySql(model.getModelSql(), modelId, null));
         }
-        System.out.println("模型结果1：" + JSON.toJSONString(modelClues));
+        System.out.println("模型结果1数量：" + modelClues.size());
         if (!modelClues.isEmpty()) {
             List<ModelClue> result = new ArrayList<>();
             for (ModelClue modelClue : modelClues) {
@@ -73,12 +81,33 @@ public class ModelClueServiceImpl implements ModelClueService {
                     result.add(modelClue);
                 }
             }
-            System.out.println("模型结果2：" + JSON.toJSONString(result));
+            System.out.println("模型结果2数量：" + modelClues.size());
             modelClueMapper.insert(result);
             modelClueRecordRepository.getBaseMapper().insert(new ModelClueRecord().setModelId(modelId).setSize(result.size()).setCreateTime(LocalDateTime.now()).setState("success"));
         } else {
             modelClueRecordRepository.getBaseMapper().insert(new ModelClueRecord().setModelId(modelId).setSize(0).setCreateTime(LocalDateTime.now()).setState("fail").setErrMsg("数据为空"));
         }
+        model.setLatestRunTime(LocalDateTime.now());
+        modelRepository.getBaseMapper().updateById(model);
+    }
+
+    public void generateCustomModels() {
+        log.info("开始执行自建模型");
+        List<Model> customModels = modelRepository.getBaseMapper().selectList(Wrappers.<Model>lambdaQuery().eq(Model::getModelingMethod, "2"));
+        if (customModels.isEmpty()) {
+            return;
+        }
+        for (Model model : customModels) {
+            if (model.getClueCycleDayOfWeek() != null
+                    && model.getClueCycleDayOfWeek().equals("week")
+                    && model.getLatestRunTime() != null
+                    && !model.getLatestRunTime().isBefore(LocalDateTime.now().minusDays(6).with(LocalTime.MIN))) {
+                log.info("模型{}距上次执行时间未满7天，跳过执行", model.getId());
+                continue;
+            }
+            generate(model.getId());
+            log.info("自建模型{}执行完毕", model.getId());
+        }
     }
 
     private void updateDepart() {
@@ -97,7 +126,7 @@ public class ModelClueServiceImpl implements ModelClueService {
         }
     }
 
-    public List<ModelClue> getModelClueBySql(String sql, Integer modelId) {
+    public List<ModelClue> getModelClueBySql(String sql, Integer modelId, Model model) {
         List<ModelClue> modelClues = new ArrayList<>();
         if (modelId == 2) { // 2号模型：行政、刑事案件受立案不及时问题监督模型
             return getModelCluesForModel2(sql, modelClues);
@@ -130,6 +159,8 @@ public class ModelClueServiceImpl implements ModelClueService {
         } else if (modelId == 105) { // 105号模型：近一月同一人同一类型问题出现3次以上的异常数据
             getModelCluesForModel105(sql, modelClues);
             return modelClues;
+        } else if (sql.equals("customModel") && model != null) {
+            return getCustomModelClues(modelId, model, modelClues);
         } else {
             System.out.println("模型" + modelId + "开始查询数据库");
             modelClues = modelClueDataMapper.selectDataByDynamicSql(sql);
@@ -194,6 +225,77 @@ public class ModelClueServiceImpl implements ModelClueService {
         return needsInsertModelClues;
     }
 
+    private @NotNull List<ModelClue> getCustomModelClues(Integer modelId, Model model, List<ModelClue> modelClues) {
+        ModelGeneration modelGeneration = modelGenerationRepository.getOne(new LambdaQueryWrapper<ModelGeneration>().eq(ModelGeneration::getModelId, modelId).last("LIMIT 1"));
+        String customModelSql = "SELECT * FROM " + model.getClueTableName() + ";";
+        System.out.println("自建模型" + modelId + "开始查询数据库");
+        List<Map<String, Object>> selectData = modelClueWdpc2Mapper.selectByUniqueKeys(customModelSql);
+        System.out.println("自建模型" + modelId + "查询数据库完毕，数量为：" + selectData.size());
+        for (Map<String, Object> map : selectData) {
+            String thingDescGeneration = modelGeneration.getThingDescGeneration();
+            String[] thingDescGenerationSplit = thingDescGeneration.split("\\^");
+            for (int i = 0; i < thingDescGenerationSplit.length; i++) {
+                for (Map.Entry<String, Object> entry : map.entrySet()) {
+                    if (thingDescGenerationSplit[i].equals(entry.getKey())) {
+                        thingDescGenerationSplit[i] = entry.getValue().toString();
+                        break;
+                    }
+                }
+            }
+            thingDescGeneration = String.join("", thingDescGenerationSplit);
+            SupExternalDepart supExternalDepart = supExternalDepartRepository.getOne(new LambdaQueryWrapper<SupExternalDepart>().eq(SupExternalDepart::getExternalId, Optional.ofNullable(map.get(modelGeneration.getInvolveDepartId()))
+                    .map(Object::toString).orElse("")).eq(SupExternalDepart::getSource, model.getClueDepartSource()).last("LIMIT 1"));
+            String involveDepartName = "";
+            String involveDepartId = "";
+            if (supExternalDepart != null && supExternalDepart.getInternalShortName() != null && !supExternalDepart.getInternalShortName().isEmpty()) {
+                involveDepartName = supExternalDepart.getInternalShortName();
+                involveDepartId = supExternalDepart.getInternalId();
+            }
+            String involvePoliceName = "";
+            String involvePoliceEmpNo = "";
+            if (modelGeneration.getInvolvePoliceName() != null) {
+                involvePoliceName = Optional.ofNullable(map.get(modelGeneration.getInvolvePoliceName()))
+                        .map(Object::toString).orElse("");            }
+            if (modelGeneration.getInvolvePoliceEmpNo() != null) {
+                involvePoliceEmpNo = Optional.ofNullable(map.get(modelGeneration.getInvolvePoliceEmpNo()))
+                        .map(Object::toString).orElse("");
+            }
+            modelClues.add(new ModelClue()
+                    .setModelId(modelId)
+                    .setInvolveDepartName(involveDepartName)
+                    .setInvolveDepartId(involveDepartId)
+                    .setInvolvePoliceName(involvePoliceName)
+                    .setInvolvePoliceEmpNo(involvePoliceEmpNo)
+                    .setThingDesc(thingDescGeneration)
+                    .setData(JSONObject.toJSONString(map))
+                    .setDistributionState("0")
+                    .setUniqueKey(map.get(model.getClueUniqueFieldName()).toString())
+                    .setNegativeIdEx(map.get(model.getClueUniqueFieldName()).toString())
+                    .setCreateTime(LocalDateTime.now()).setUpdateTime(LocalDateTime.now()));
+        }
+        System.out.println("自建模型" + modelId + "的modelClues数量为：" + modelClues.size());
+        List<String> uniqueKeys = modelClues.stream().map(ModelClue::getUniqueKey).filter(k -> k != null && !k.isEmpty()).toList();
+        List<String> oldUniqueKeys = modelClueRepository.listObjs(new LambdaQueryWrapper<ModelClue>().select(ModelClue::getUniqueKey).ne(ModelClue::getDistributionState, "2").eq(ModelClue::getModelId, modelId), String::valueOf);
+        List<String> changeStatusIds = oldUniqueKeys.stream().filter(item -> !uniqueKeys.contains(item)).toList();
+        if (!changeStatusIds.isEmpty()) {
+            modelClueMapper.update(null, new LambdaUpdateWrapper<ModelClue>().in(ModelClue::getUniqueKey, changeStatusIds).set(ModelClue::getDistributionState, "2"));
+        }
+        List<String> needsInsertIds = new ArrayList<>();
+        for (String uniqueKey : uniqueKeys) {
+            if (oldUniqueKeys.isEmpty()) {
+                needsInsertIds.addAll(uniqueKeys);
+                break;
+            }
+            if (!oldUniqueKeys.contains(uniqueKey)) {
+                needsInsertIds.add(uniqueKey);
+            }
+        }
+        if (needsInsertIds.isEmpty()) {
+            return new ArrayList<>();
+        }
+        return modelClues.stream().filter(item -> needsInsertIds.contains(item.getUniqueKey())).toList();
+    }
+
     private void getModelCluesForModel7(String param, List<ModelClue> modelClues) {
 
 //        List<Object> jsonStrings = redisTemplate.opsForList().range("call:point", 0, -1);

src/main/java/com/biutag/supervisiondata/task/TaskService.java

@@ -1,21 +1,25 @@
 package com.biutag.supervisiondata.task;
 
-import com.biutag.supervisiondata.common.interfaces.CompletableFutureUtil;
-import com.biutag.supervisiondata.service.*;
-import lombok.AllArgsConstructor;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+import com.biutag.supervisiondata.common.interfaces.CompletableFutureUtil;
 import com.biutag.supervisiondata.pojo.domain.TaskParamDomain;
+import com.biutag.supervisiondata.pojo.entity.mine.Model;
 import com.biutag.supervisiondata.pojo.entity.mine.RiskModelTaskClue;
 import com.biutag.supervisiondata.pojo.entity.mine.RiskPersonal;
+import com.biutag.supervisiondata.repository.ModelRepository;
 import com.biutag.supervisiondata.repository.RiskModelTaskClueRepository;
 import com.biutag.supervisiondata.repository.RiskPersonalRepository;
+import com.biutag.supervisiondata.service.*;
 import jakarta.annotation.Resource;
+import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import java.time.LocalDate;
 import java.time.LocalDateTime;
+import java.time.LocalTime;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -56,6 +60,9 @@ public class TaskService {
     @Resource
     private RiskPersonalService riskPersonalService;
 
+    @Resource
+    private ModelRepository modelRepository;
+
     /**
      * 1点半更新人员
      */
@@ -234,4 +241,27 @@ public class TaskService {
         modelClueService.generate(105);
         log.info("模型105执行完毕");
     }
+
+    /**
+     * 4点更新自建模型
+     */
+    @Scheduled(cron = "0 0 4 * * ?")
+    public void runCustomModels() {
+        log.info("开始执行自建模型");
+        List<Model> customModels = modelRepository.getBaseMapper().selectList(Wrappers.<Model>lambdaQuery().eq(Model::getModelType, "2"));
+        if (customModels.isEmpty()) {
+            return;
+        }
+        for (Model model : customModels) {
+            if (model.getClueCycleDayOfWeek() != null
+                    && model.getClueCycleDayOfWeek().equals("week")
+                    && model.getLatestRunTime() != null
+                    && !model.getLatestRunTime().isBefore(LocalDateTime.now().minusDays(6).with(LocalTime.MIN))) {
+                log.info("模型{}距上次执行时间未满7天，跳过执行", model.getId());
+                continue;
+            }
+            modelClueService.generate(model.getId());
+            log.info("自建模型{}执行完毕", model.getId());
+        }
+    }
 }