diff --git a/build.gradle b/build.gradle index a93d4ae8d0f0410dc355b9fa00466345ce313c3b..90ad297ceaf9a2a9b672a6b152a39f4e55978fd7 100644 --- a/build.gradle +++ b/build.gradle @@ -60,8 +60,9 @@ dependencies { implementation group: 'org.apache.cxf', name: 'cxf-core', version: '3.3.4' implementation 'org.bdware.doip:doip-audit-tool:1.6.0' implementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' + implementation "org.bdware.bdcontract:ypk-deploy-tool:0.7.5" testImplementation "org.bdware.bdcontract:ypk-deploy-tool:0.7.5" - testImplementation "org.bdware.bdcontract:sdk-java:1.0.2" + implementation "org.bdware.bdcontract:sdk-java:1.0.2" } group = "org.bdware.datanet.router" diff --git a/src/main/java/org/bdware/datanet/router/Config.java b/src/main/java/org/bdware/datanet/router/Config.java index 02f1b5d2dc9c7d533c25be31424975ccd6092928..8acd627db3ba532ac080dbba9e488bd2642553eb 100644 --- a/src/main/java/org/bdware/datanet/router/Config.java +++ b/src/main/java/org/bdware/datanet/router/Config.java @@ -108,9 +108,7 @@ public class Config { // @fanbo public String getPubKey() { if (configStorage.get("pubKey") == null) { - // test return "04da01345770b7e09d4774bf6c0395399b18814aa0b7b158f64b634b8f3d628d9964af6523835225af11e467271f4969d67bf90b32eaa09f517c79b2d1f9b8a926"; - // return null; } String pubKey = configStorage.get("pubKey"); return pubKey; @@ -120,6 +118,15 @@ public class Config { configStorage.put("pubKey", pubKey); } + public String getPrivKey(){ + String privKey = configStorage.get("privKey"); + return privKey; + } + + public void setPrivKey(String privKey) { + configStorage.put("privKey", privKey); + } + public void setScheme(String scheme) { configStorage.put("scheme", scheme); } diff --git a/src/main/java/org/bdware/datanet/router/DoidDAO.java b/src/main/java/org/bdware/datanet/router/DoidDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..e785df27ec2c2b0eb25fb62dda0594fcd1327415 --- /dev/null +++ b/src/main/java/org/bdware/datanet/router/DoidDAO.java @@ -0,0 +1,1076 @@ +package org.bdware.datanet.router; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.bdware.sc.boundry.utils.RocksDBUtil; +import org.bdware.datanet.router.util.JsonUtils; +import org.rocksdb.*; + +import javax.json.Json; +import javax.print.DocFlavor.STRING; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.bdware.datanet.router.RowStore.LOGGER; + +public class DoidDAO { + static { RocksDB.loadLibrary();} + + // 初始化标志 + private static volatile boolean initialized = false; + private static final Object initLock = new Object(); + + // 主表 + static ColumnFamilyHandle defaultCF; // 保留 + static ColumnFamilyHandle doidCF; // 存储 data 和 code* 类型数据 + static ColumnFamilyHandle prefixCF; // 存储 prefix 类型数据 + static ColumnFamilyHandle repoCF; // 存储 system 类型数据 + static ColumnFamilyHandle countCF; // 存储计数信息 + + // 索引表 + static ColumnFamilyHandle indexTimeIdCF; // 索引1: 数据标识时间索引 + static ColumnFamilyHandle indexTimePrefixCF; // 索引2: 前缀时间索引 + static ColumnFamilyHandle indexTimeRepoCF; // 索引3: 后缀时间索引 + static ColumnFamilyHandle indexTimeCodeCF; // 索引4: code时间索引 + + // 辅助表 + static ColumnFamilyHandle namePubkeyVerifyCF; // 用来存储具有连通属性的doid + static ColumnFamilyHandle lrsStatisticsCF; // 用来存储LRS统计信息 + + static RocksDB db; + + /** + * 静态初始化方法 - 优雅地初始化DoidDAO + * 使用双重检查锁定模式确保线程安全和性能 + * @throws RocksDBException 如果初始化失败 + */ + public static void initialize() throws RocksDBException { + if (!initialized) { + synchronized (initLock) { + if (!initialized) { + try { + LOGGER.info("Initializing DoidDAO..."); + new DoidDAO(); // 调用构造函数进行实际初始化 + initialized = true; + + // 添加shutdown hook,确保程序退出时优雅关闭数据库 + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LOGGER.info("Shutdown hook triggered, closing DoidDAO..."); + close(); + }, "DoidDAO-Shutdown-Hook")); + } catch (RocksDBException e) { + LOGGER.error("Failed to initialize DoidDAO: {}", e.getMessage(), e); + throw e; + } + } + } + } + } + + /** + * 检查DoidDAO是否已初始化 + * @return true如果已初始化,false否则 + */ + public static boolean isInitialized() { + return initialized; + } + + public DoidDAO() throws RocksDBException { + String DB_PATH = "./doStorage"; + // 定义 ColumnFamilyDescriptor 列表(包含默认CF) + List cfDescriptors = Arrays.asList( + // 默认 CF,RocksDB示例必须至少有一个 + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, new ColumnFamilyOptions()), + // 主表 + new ColumnFamilyDescriptor("CF_doid".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeUniversalStyleCompaction()), + new ColumnFamilyDescriptor("CF_prefix".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeUniversalStyleCompaction()), + new ColumnFamilyDescriptor("CF_repo".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeUniversalStyleCompaction()), + new ColumnFamilyDescriptor("CF_count".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeUniversalStyleCompaction().setMergeOperator(new UInt64AddOperator())), + // 索引表 + new ColumnFamilyDescriptor("CF_index_time_id".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeLevelStyleCompaction()), + new ColumnFamilyDescriptor("CF_index_time_prefixId".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeLevelStyleCompaction()), + new ColumnFamilyDescriptor("CF_index_time_repo".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeLevelStyleCompaction()), + new ColumnFamilyDescriptor("CF_index_time_code".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeLevelStyleCompaction()), + // 辅助表 + new ColumnFamilyDescriptor("CF_name_pubkey_verify".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeLevelStyleCompaction()), + new ColumnFamilyDescriptor("CF_lrs_statistics".getBytes(StandardCharsets.UTF_8), new ColumnFamilyOptions().optimizeLevelStyleCompaction()) + ); + + // 2. 打开 RocksDB,并创建缺失的 CF + DBOptions dbOpts = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + + List cfHandles = new ArrayList<>(); + db = RocksDB.open(dbOpts, DB_PATH, cfDescriptors, cfHandles); + + // 3. 拿到各个 CF 的句柄 + defaultCF = cfHandles.get(0); + doidCF = cfHandles.get(1); + prefixCF = cfHandles.get(2); + repoCF = cfHandles.get(3); + countCF = cfHandles.get(4); + indexTimeIdCF = cfHandles.get(5); + indexTimePrefixCF = cfHandles.get(6); + indexTimeRepoCF = cfHandles.get(7); + indexTimeCodeCF = cfHandles.get(8); + namePubkeyVerifyCF = cfHandles.get(9); + lrsStatisticsCF = cfHandles.get(10); + } + + public static class ReadOptionsResult { + private final ReadOptions readOptions; + private final byte[] lowerBytes; + private final byte[] upperKeyBytes; + + public ReadOptionsResult(ReadOptions readOptions, byte[] lowerBytes, byte[] upperKeyBytes) { + this.readOptions = readOptions; + this.lowerBytes = lowerBytes; + this.upperKeyBytes = upperKeyBytes; + } + public ReadOptions getReadOptions() { + return readOptions; + } + public byte[] getLowerBytes() { + return lowerBytes; + } + public byte[] getUpperKeyBytes() { + return upperKeyBytes; + } + } + + /** + * 构建时间范围查询的 ReadOptions + * 新设计中索引键格式为: {{timestamp}}_{{doidHash}} 或 {{timestamp}}{{type}}_{{doidHash}} + */ + private static ReadOptionsResult buildTimeRangeReadOptions(Long startTs, Long endTs, String typePrefix) { + // 构造下界(inclusive) + String lowerKey; + if (startTs != null) { + lowerKey = (typePrefix != null ? String.format("%013d", startTs) + typePrefix : String.format("%013d", startTs)); + } else { + lowerKey = (typePrefix != null ? String.format("%013d", 0) + typePrefix : String.format("%013d", 0)); + } + byte[] lowerBytes = lowerKey.getBytes(StandardCharsets.UTF_8); + + // 构造上界(exclusive) + String upperKey; + if (endTs != null) { + upperKey = (typePrefix != null ? String.format("%013d", endTs + 1) + typePrefix : String.format("%013d", endTs + 1)); + } else { + upperKey = (typePrefix != null ? String.format("%013d", Long.MAX_VALUE) + typePrefix : String.format("%013d", Long.MAX_VALUE)); + } + byte[] upperKeyBytes = upperKey.getBytes(StandardCharsets.UTF_8); + + ReadOptions ro = new ReadOptions() + .setPrefixSameAsStart(true) + .setIterateLowerBound(new Slice(lowerBytes)) + .setIterateUpperBound(new Slice(upperKeyBytes)); + + return new ReadOptionsResult(ro, lowerBytes, upperKeyBytes); + } + + /** + * @param timestamp 时间戳 + * @param doidHash DOID的哈希值 + * @return 索引键字符串 + */ + private static String buildIndexKey(long timestamp, int doidHash) { + StringBuilder sb = new StringBuilder(32); // 预分配足够容量 + sb.append(String.format("%013d", timestamp)) + .append("_") + .append(doidHash); + return sb.toString(); + } + + /** + * @param timestamp 时间戳 + * @param type 类型前缀 + * @param doidHash DOID的哈希值 + * @return 索引键字符串 + */ + private static String buildTypedIndexKey(long timestamp, String type, int doidHash) { + StringBuilder sb = new StringBuilder(64); // 预分配足够容量 + sb.append(String.format("%013d", timestamp)) + .append(type) + .append("_") + .append(doidHash); + return sb.toString(); + } + + /** + * 批量更新计数 + */ + private static void updateCount(WriteBatch batch, String keyType, int delta) throws RocksDBException { + LOGGER.info("[updateCount]: {} {}", keyType, delta); + try{ + if (keyType.contains("/")){ + keyType = keyType.substring(0,keyType.indexOf("/")); + } + byte[] deltaBytes = ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(delta).array(); + batch.merge(countCF, keyType.getBytes(),deltaBytes); + } catch (Exception e) { + LOGGER.error("[updateCount error] {}", e.getMessage()); + e.printStackTrace(); + } + } + /** + * 获取指定类型的计数 + */ + static public int getCount(String keyType) { + try { + if (keyType.contains("/")) { + keyType = keyType.substring(0, keyType.indexOf("/")); + } + byte[] keyBytes = keyType.getBytes(StandardCharsets.UTF_8); + byte[] countBytes = db.get(countCF, keyBytes); + // 要求至少 8 字节 + if (countBytes != null && countBytes.length >= Long.BYTES) { + long cnt = ByteBuffer + .wrap(countBytes) + .order(ByteOrder.LITTLE_ENDIAN) // 与写入时一致 + .getLong(); + return (int) cnt; + } + } catch (RocksDBException e) { + e.printStackTrace(); + } + return 0; + } + + /** + * 存储数据对象 - 使用 BatchWrite 实现原子操作 + * 根据类型存储到不同的表中,并建立相应的索引 + */ + static public void putDO(String doid, JsonObject jo) throws Exception{ + boolean skipAddCount = isExistedDoid(doid); // 存在就跳过更新数量 + long timestamp = System.currentTimeMillis(); + if (jo.has("_updateTimestamp")) { + String timestampStr = JsonUtils.getStringSafely(jo, "_updateTimestamp", String.valueOf(timestamp)); + try { + Long timestamp1 = Long.parseLong(timestampStr); + timestamp = timestamp1; + } catch (NumberFormatException e) { + // 如果解析失败,使用当前时间戳 + jo.addProperty("_updateTimestamp",timestamp); + } + } else { + jo.addProperty("_updateTimestamp", timestamp); + } + String type = JsonUtils.getStringSafely(jo, "type", "data"); + + try(WriteBatch batch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions();) { + String jsonData = jo.toString(); + byte[] dataBytes = jsonData.getBytes(StandardCharsets.UTF_8); + String indexKey = buildIndexKey(timestamp, doid.hashCode()); + byte[] indexKeyBytes = indexKey.getBytes(StandardCharsets.UTF_8); + byte[] doidBytes = doid.getBytes(StandardCharsets.UTF_8); + + // 根据类型存储到不同的表 + if (type.equals("prefix")) { + batch.put(prefixCF, doidBytes, dataBytes); + // 建立索引2: CF_index_time_prefixId + batch.put(indexTimePrefixCF, indexKeyBytes, doidBytes); + + } else if (type.startsWith("system")) { + // 存储到 CF_repo 表 + batch.put(repoCF, doidBytes, dataBytes); + // 建立索引3: CF_index_time_repo + String repoIndexKey = buildTypedIndexKey(timestamp, type, doid.hashCode()); + batch.put(indexTimeRepoCF, repoIndexKey.getBytes(StandardCharsets.UTF_8), doidBytes); + + } else if (type.startsWith("code")) { + // 存储到 CF_doid 表(code* 类型) + batch.put(doidCF, doidBytes, dataBytes); + // 建立索引4: CF_index_time_code + String codeIndexKey = buildTypedIndexKey(timestamp, type, doid.hashCode()); + batch.put(indexTimeCodeCF, codeIndexKey.getBytes(StandardCharsets.UTF_8), doidBytes); + + } else { + // 默认存储到 CF_doid 表(data 类型) + batch.put(doidCF, doidBytes, dataBytes); + // 建立索引1: CF_index_time_id(不包含code) + if (!type.startsWith("code")) { + batch.put(indexTimeIdCF, indexKeyBytes, doidBytes); + } + } + + if (!skipAddCount) { + updateCount(batch, type, 1); + } + + // 原子性提交所有操作 + db.write(writeOptions, batch); + }catch (Exception e){ + e.printStackTrace(); + LOGGER.error("[DAO ERROR] : putDO failed: {} " , e.getMessage()); + throw e; + } + } + /** + * 存储数据对象 - 使用 BatchWrite 实现原子操作 + * 根据类型存储到不同的表中,并建立相应的索引 + */ + static public void putNamePubkey(String key, String doid) { + try { + db.put(namePubkeyVerifyCF, key.getBytes(), doid.getBytes()); + } catch (RocksDBException e) { + e.printStackTrace(); + LOGGER.error("[DAO ERROR] : putNamePubkey failed: " + e.getMessage()); + } + } + + static public String getNamePubkey(String key) { + try { + byte[] doidBytes = db.get(namePubkeyVerifyCF, key.getBytes()); + if (doidBytes == null) { + return null; // 键不存在时返回null + } + return new String(doidBytes, StandardCharsets.UTF_8); + } catch (RocksDBException e) { + LOGGER.error("[DAO ERROR] : getNamePubkey failed: {} " , e.getMessage()); + e.printStackTrace(); + } + return null; + } + + static public void deleteNamePubkey(String key) { + try { + db.delete(namePubkeyVerifyCF, key.getBytes()); + } catch (RocksDBException e) { + e.printStackTrace(); + LOGGER.error("[DAO ERROR] : deleteNamePubkey failed: " + e.getMessage()); + } + } + + /** + * 获取数据对象 - 使用 MultiGet 优化版本 + * 一次性查询所有列族,减少底层查找次数和磁盘I/O + */ + static public String getDO(String doid) { + try { + byte[] doidBytes = doid.getBytes(StandardCharsets.UTF_8); + // 使用 MultiGet 一次性查询所有相关列族 + List cfList = Arrays.asList(doidCF, prefixCF, repoCF); + List keyList = Arrays.asList(doidBytes, doidBytes, doidBytes); + + List results = db.multiGetAsList(cfList, keyList); + + for (byte[] result : results) { + if (result != null) { + return new String(result, StandardCharsets.UTF_8); + } + } + + } catch (RocksDBException e) { + e.printStackTrace(); + } + return null; + } + + /** + * 批量获取数据对象 - 利用 MultiGet 的批量优势 + * 一次性查询多个 doid,显著减少网络和磁盘I/O开销 + */ + static public List batchGetDO(List doids) { + List results = new ArrayList<>(doids.size()); + if (doids.isEmpty()) { + return results; + } + + try { + // 准备批量查询的参数 + List cfList = new ArrayList<>(); + List keyList = new ArrayList<>(); + + // 为每个 doid 在每个列族中都准备一个查询 + for (String doid : doids) { + byte[] doidBytes = doid.getBytes(StandardCharsets.UTF_8); + // 按优先级顺序:doidCF, prefixCF, repoCF + cfList.add(doidCF); + cfList.add(prefixCF); + cfList.add(repoCF); + keyList.add(doidBytes); + keyList.add(doidBytes); + keyList.add(doidBytes); + } + + // 执行批量查询 + List batchResults = db.multiGetAsList(cfList, keyList); + LOGGER.info("[DAO] : batchGetDO: {},size:{}", batchResults, batchResults.size()); + // 处理结果:每个 doid 对应 3 个查询结果 + for (int i = 0; i < doids.size(); i++) { + byte[] result = null; + // 按优先级检查结果 + for (int j = 0; j < 3; j++) { + int index = i * 3 + j; + if (batchResults.get(index) != null) { + result = batchResults.get(index); + break; + } + } + results.add(result); + } + LOGGER.info("[DAO] : batchGetDO: {},size:{}", results, results.size()); + + } catch (RocksDBException e) { + e.printStackTrace(); + // 发生异常时返回空结果列表 + for (int i = 0; i < doids.size(); i++) { + results.add(null); + } + } + + return results; + } + + static public void deleteDO(String doid){ + String data = getDO(doid); + if (data.isEmpty()) { + return; + } + + JsonObject jo = JsonParser.parseString(data).getAsJsonObject(); + String type = JsonUtils.getStringSafely(jo, "type", "data"); + long timestamp = JsonUtils.getLongSafely(jo, "_updateTimestamp", System.currentTimeMillis()); + + try (WriteBatch batch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { + byte[] doidBytes = doid.getBytes(StandardCharsets.UTF_8); + String indexKey = buildIndexKey(timestamp, doid.hashCode()); + byte[] indexKeyBytes = indexKey.getBytes(StandardCharsets.UTF_8); + + // 根据类型从不同的表中删除 + if (type.equals("prefix")) { + // 从 CF_prefix 表删除 + batch.delete(prefixCF, doidBytes); + // 删除索引2: CF_index_time_prefixId + batch.delete(indexTimePrefixCF, indexKeyBytes); + + } else if (type.startsWith("system")) { + // 从 CF_repo 表删除 + batch.delete(repoCF, doidBytes); + // 删除索引3: CF_index_time_repo + String repoIndexKey = buildTypedIndexKey(timestamp, type, doid.hashCode()); + batch.delete(indexTimeRepoCF, repoIndexKey.getBytes(StandardCharsets.UTF_8)); + + } else if (type.startsWith("code")) { + // 从 CF_doid 表删除(code* 类型) + batch.delete(doidCF, doidBytes); + // 删除索引4: CF_index_time_code + String codeIndexKey = buildTypedIndexKey(timestamp, type, doid.hashCode()); + batch.delete(indexTimeCodeCF, codeIndexKey.getBytes(StandardCharsets.UTF_8)); + + } else { + // 从 CF_doid 表删除(data 类型) + batch.delete(doidCF, doidBytes); + // 删除索引1: CF_index_time_id(不包含code) + if (!type.startsWith("code")) { + batch.delete(indexTimeIdCF, indexKeyBytes); + } + } + + // 批量更新计数 - 支持层级计数 + updateCount(batch, type, -1); + + // 原子性提交所有操作 + db.write(writeOptions, batch); + + } catch (RocksDBException e) { + e.printStackTrace(); + throw new RuntimeException("Failed to delete data object: " + doid, e); + } + } + + /** + * 查询场景1: 数据标识列表(时间正序或倒序),不包含code + * 使用索引1: CF_index_time_id + */ + static public JsonObject queryDataList(Long startTs, Long endTs, int offset, int count, boolean ascending) throws Exception{ + JsonObject ret = new JsonObject(); + try { + List doids = queryDoidsByTimeIndex(indexTimeIdCF, startTs, endTs, offset, count, ascending, null); + if (!doids.isEmpty()) { + // 使用批量获取优化性能 + List dataList = batchGetDO(doids); + int d = 0; + for (byte[] data : dataList) { + if (data != null) { + ret.add(doids.get(d),JsonParser.parseString(new String(data, StandardCharsets.UTF_8))); + } + d++; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return ret; + } + + static public JsonArray queryDataDoidList(Long startTs, Long endTs, int offset, int count, boolean ascending) throws Exception{ + JsonArray ret = new JsonArray(); + try { + List doids = queryDoidsByTimeIndex(indexTimeIdCF, startTs, endTs, offset, count, ascending, null); + for (String doid : doids) { + ret.add(doid); + } + } catch (Exception e) { + e.printStackTrace(); + } + return ret; + } + + /** + * 查询场景2: 前缀列表(type = prefix 时间正序或倒序) + * 使用索引2: CF_index_time_prefixId + */ + static public JsonObject queryPrefixList(Long startTs, Long endTs, int offset, int count, boolean ascending) throws Exception{ + JsonObject ret = new JsonObject(); + try { + List doids = queryDoidsByTimeIndex(indexTimePrefixCF, startTs, endTs, offset, count, ascending, null); + if (!doids.isEmpty()) { + // 使用批量获取优化性能 + List dataList = batchGetDO(doids); + int d = 0; + for (byte[] data : dataList) { + if (data != null) { + ret.add(doids.get(d),JsonParser.parseString(new String(data, StandardCharsets.UTF_8))); + } + d++; + } + } + } catch (Exception e) { + LOGGER.error("[queryPrefixList] error: {}", e.getMessage()); + throw e; + } + return ret; + } + + static public JsonObject queryPrefixListInSearchMode(String doid,int offset, int count, boolean ascending) throws Exception{ + JsonObject ret = new JsonObject(); + + byte[] lowerBytes = doid.getBytes(StandardCharsets.UTF_8); + String upperKey = doid + "\uFFFF"; + byte[] upperBytes = upperKey.getBytes(StandardCharsets.UTF_8); + + ReadOptions ro = new ReadOptions().setPrefixSameAsStart(true).setIterateLowerBound(new Slice(lowerBytes)). + setIterateUpperBound(new Slice(upperBytes)); + + RocksIterator iter = db.newIterator(prefixCF,ro); + try { + if (ascending){ + iter.seek(lowerBytes); + }else{ + iter.seekForPrev(upperBytes); + } + int skip = 0; + while( skip < offset && iter.isValid()){ + if (ascending){ + iter.next(); + }else{ + iter.prev(); + } + skip++; + } + + while(count > 0 && iter.isValid()){ + String key = new String(iter.key(),StandardCharsets.UTF_8); + JsonObject value = JsonParser.parseString(new String(iter.value(),StandardCharsets.UTF_8)).getAsJsonObject(); + ret.add(key,value); + if (ascending){ + iter.next(); + }else{ + iter.prev(); + } + count --; + } + } catch (Exception e) { + LOGGER.error("[queryPrefixListInSearchMode] error: {}", e.getMessage()); + throw e; + }finally { + iter.close(); + } + return ret; + } + + + /** + * 查询场景3: 后缀列表(type = system开头 时间正序或倒序) + * 使用索引3: CF_index_time_repo + */ + static public JsonObject queryRepoList(String systemType, Long startTs, Long endTs, int offset, int count, boolean ascending) throws Exception{ + JsonObject ret = new JsonObject(); + List doids = queryDoidsByTimeIndex(indexTimeRepoCF, startTs, endTs, offset, count, ascending, systemType); + if (!doids.isEmpty()) { + // 使用批量获取优化性能 + List dataList = batchGetDO(doids); + int d = 0; + for (byte[] data : dataList) { + if (data != null) { + ret.add(doids.get(d),JsonParser.parseString(new String(data, StandardCharsets.UTF_8))); + } + d++; + } + } + return ret; + } + static public JsonObject queryRepoListInSearchMode(String doid,int offset, int count, boolean ascending) throws Exception{ + JsonObject ret = new JsonObject(); + + byte[] lowerBytes = doid.getBytes(StandardCharsets.UTF_8); + String upperKey = doid + "\uFFFF"; + byte[] upperBytes = upperKey.getBytes(StandardCharsets.UTF_8); + + ReadOptions ro = new ReadOptions().setPrefixSameAsStart(true).setIterateLowerBound(new Slice(lowerBytes)). + setIterateUpperBound(new Slice(upperBytes)); + + RocksIterator iter = db.newIterator(repoCF,ro); + try { + if (ascending){ + iter.seek(lowerBytes); + }else{ + iter.seekForPrev(upperBytes); + } + int skip = 0; + while(skip < offset && iter.isValid()){ + if (ascending){ + iter.next(); + }else{ + iter.prev(); + } + skip++; + } + + int taken = 0; + while(taken < count && iter.isValid()){ + String key = new String(iter.key(),StandardCharsets.UTF_8); + JsonObject value = JsonParser.parseString(new String(iter.value(),StandardCharsets.UTF_8)).getAsJsonObject(); + ret.add(key,value); + if (ascending){ + iter.next(); + }else{ + iter.prev(); + } + taken++; + } + } catch (Exception e) { + LOGGER.error("[queryRepoListInSearchMode] error: {}", e.getMessage()); + throw e; + }finally { + iter.close(); + } + return ret; + } + + static public JsonObject listPrefix() { + JsonObject ret = new JsonObject(); + JsonObject data = new JsonObject(); + RocksIterator iter = db.newIterator(prefixCF); + try { + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + String key = new String(iter.key(), StandardCharsets.UTF_8); + data.add(key,JsonParser.parseString(new String(iter.value(), StandardCharsets.UTF_8))); + } + ret.add("data", data); + ret.addProperty("result","success"); + ret.addProperty("count", data.size()); + } catch (Exception e) { + ret.addProperty("result", "failed"); + e.printStackTrace(); + }finally{ + iter.close(); + } + return ret; + } + + static public JsonObject listRepo() { + JsonObject ret = new JsonObject(); + JsonObject data = new JsonObject(); + RocksIterator iter = db.newIterator(repoCF); + try { + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + String key = new String(iter.key(), StandardCharsets.UTF_8); + data.add(key,JsonParser.parseString(new String(iter.value(), StandardCharsets.UTF_8))); + } + ret.add("data", data); + ret.addProperty("result","success"); + ret.addProperty("count", data.size()); + } catch (Exception e) { + ret.addProperty("result", "failed"); + e.printStackTrace(); + }finally{ + iter.close(); + } + return ret; + } + + static public JsonObject listNamePubkey(){ + JsonObject ret = new JsonObject(); + RocksIterator iter = db.newIterator(namePubkeyVerifyCF); + try { + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + String key = new String(iter.key(), StandardCharsets.UTF_8); + ret.addProperty(key,new String(iter.value(), StandardCharsets.UTF_8)); + } + } catch (Exception e) { + ret.addProperty("result", "failed"); + e.printStackTrace(); + }finally{ + iter.close(); + } + return ret; + } + + /** + * 查询场景4: code开头的数据列表查询(type = code开头 时间正序或倒序) + * 使用索引4: CF_index_time_code + */ + static public JsonObject queryCodeList(String codeType, Long startTs, Long endTs, int offset, int count, boolean ascending) throws Exception{ + JsonObject ret = new JsonObject(); + try { + List doids = queryDoidsByTimeIndex(indexTimeCodeCF, startTs, endTs, offset, count, ascending, codeType); + if (!doids.isEmpty()) { + // 使用批量获取优化性能 + List dataList = batchGetDO(doids); + int d = 0; + for (byte[] data : dataList) { + if (data != null) { + ret.add(doids.get( d),JsonParser.parseString(new String(data, StandardCharsets.UTF_8))); + } + d ++; + } + } + } catch (Exception e) { + LOGGER.error("[DAO ERROR queryCodeList] error: {}", e.getMessage()); + e.printStackTrace(); + } + return ret; + } + + /** + * 通用的时间索引查询方法 + */ + private static List queryDoidsByTimeIndex(ColumnFamilyHandle indexCF, Long startTs, Long endTs, + int offset, int count, boolean ascending, String typePrefix) throws Exception { + ReadOptionsResult ro = buildTimeRangeReadOptions(startTs, endTs, typePrefix); + List result = new ArrayList<>(count); + RocksIterator iter = db.newIterator(indexCF, ro.getReadOptions()); + try { + if (ascending) { + iter.seek(ro.getLowerBytes()); + } else { + iter.seekForPrev(ro.getUpperKeyBytes()); + } + + // 跳过 offset + int skipped = 0; + while (skipped < offset && iter.isValid()) { + if (ascending) { + iter.next(); + } else { + iter.prev(); + } + skipped++; + } + + // 收集结果 + while (count > 0 && iter.isValid()) { + result.add(new String(iter.value(), StandardCharsets.UTF_8)); + if (ascending) { + iter.next(); + } else { + iter.prev(); + } + count--; + } + }catch (Exception e){ + LOGGER.error("[DAO ERROR queryDoidsByTimeIndex] error: {}", e.getMessage()); + throw e; + }finally{ + iter.close(); + } + return result; + } + + // 保持向后兼容的查询方法 + static public JsonObject query(String type, Long startTs, Long endTs, int offset, int count, boolean ascending) throws Exception{ + if (type == null || type.isEmpty() || type.equals("data")) { + return queryDataList(startTs, endTs, offset, count, ascending); + } else if (type.equals("prefix")) { + return queryPrefixList(startTs, endTs, offset, count, ascending); + } else if (type.startsWith("system")) { + return queryRepoList(type, startTs, endTs, offset, count, ascending); + } else if (type.startsWith("code")) { + return queryCodeList(type, startTs, endTs, offset, count, ascending); + } else { + return queryDataList(startTs, endTs, offset, count, ascending); + } + } + + + + static public boolean isExistedDoid(String identifier) { + String content = DoidDAO.getDO(identifier); + return content != null; + } + + /** + * 检查公钥是否被其他前缀使用 + * @param pubKey 要检查的公钥 + * @param excludeDoid 要排除的DOID(当前正在更新的DOID) + * @return true 如果公钥被其他前缀使用,false 否则 + */ + static public boolean isPubKeyUsedByOtherPrefix(String pubKey, String excludeDoid) { + if (pubKey == null || pubKey.isEmpty()) { + return false; + } + + RocksIterator iter = db.newIterator(prefixCF); + try { + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + String doid = new String(iter.key(), StandardCharsets.UTF_8); + + // 跳过要排除的DOID + if (excludeDoid != null && excludeDoid.equals(doid)) { + continue; + } + + try { + String jsonData = new String(iter.value(), StandardCharsets.UTF_8); + JsonObject prefixData = JsonParser.parseString(jsonData).getAsJsonObject(); + + if (prefixData.has("pubKey")) { + String existingPubKey = prefixData.get("pubKey").getAsString(); + if (pubKey.equals(existingPubKey)) { + return true; + } + } + } catch (Exception e) { + LOGGER.warn("[isPubKeyUsedByOtherPrefix] Failed to parse prefix data for doid: {}, error: {}", doid, e.getMessage()); + } + } + } catch (Exception e) { + LOGGER.error("[isPubKeyUsedByOtherPrefix] Error during iteration: {}", e.getMessage()); + } finally { + iter.close(); + } + + return false; + } + + + public static Object clearAllStorage() { + JsonObject ret = new JsonObject(); + ret.addProperty("result", "success"); + try (WriteBatch batch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { + // 清空所有主表和索引表 + clearColumnFamily(batch, doidCF); + clearColumnFamily(batch, prefixCF); + clearColumnFamily(batch, repoCF); + clearColumnFamily(batch, indexTimeIdCF); + clearColumnFamily(batch, indexTimePrefixCF); + clearColumnFamily(batch, indexTimeRepoCF); + clearColumnFamily(batch, indexTimeCodeCF); + clearColumnFamily(batch, namePubkeyVerifyCF); + clearColumnFamily(batch, lrsStatisticsCF); + clearColumnFamily(batch, countCF); + + // 原子性提交所有删除操作 + db.write(writeOptions, batch); + } catch (Exception e) { + ret.addProperty("result", "failed"); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + e.printStackTrace(new PrintStream(bo)); + ret.addProperty("errorMessage", bo.toString()); + } + return ret; + } + + /** + * 清空指定列族中的所有数据 + * + * @param batch WriteBatch对象 + * @param cf 要清空的列族 + * @throws RocksDBException + */ + private static void clearColumnFamily(WriteBatch batch, ColumnFamilyHandle cf) throws RocksDBException { + RocksIterator iter = db.newIterator(cf); + try { + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + batch.delete(cf, iter.key()); + } + }catch (Exception e){ + e.printStackTrace(); + }finally{ + iter.close(); + } + } + + // ==================== LRSStatistics 相关方法 ==================== + /** + * 获取LRS统计信息 + * @param doid 数据对象标识符 + * @return LRS统计信息的JSON字符串,如果不存在则返回null + */ + static public String getLRSStatistics(String doid) { + try { + byte[] doidBytes = doid.getBytes(StandardCharsets.UTF_8); + byte[] statisticsBytes = db.get(lrsStatisticsCF, doidBytes); + if (statisticsBytes != null) { + return new String(statisticsBytes, StandardCharsets.UTF_8); + } + } catch (RocksDBException e) { + LOGGER.error("[DAO ERROR] getLRSStatistics failed for doid: {}, error: {}", doid, e.getMessage()); + e.printStackTrace(); + } + return null; + } + + /** + * 获取所有LRS统计信息列表 + * @return 包含所有LRS统计信息的列表 + */ + static public List getLRSStatisticsList() { + List result = new ArrayList<>(); + RocksIterator iter = db.newIterator(lrsStatisticsCF); + try { + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + String statisticsData = new String(iter.value(), StandardCharsets.UTF_8); + result.add(statisticsData); + } + } catch (Exception e) { + LOGGER.error("[DAO ERROR] getLRSStatisticsList failed: {}", e.getMessage()); + e.printStackTrace(); + } finally { + iter.close(); + } + return result; + } + + /** + * 存储LRS统计信息 + * @param doid 数据对象标识符 + * @param statisticsData LRS统计信息的JSON字符串 + * @return 操作是否成功 + */ + static public boolean storageLRSStatistics(String doid, String statisticsData) { + try { + byte[] doidBytes = doid.getBytes(StandardCharsets.UTF_8); + byte[] dataBytes = statisticsData.getBytes(StandardCharsets.UTF_8); + db.put(lrsStatisticsCF, doidBytes, dataBytes); + return true; + } catch (RocksDBException e) { + LOGGER.error("[DAO ERROR] storageLRSStatistics failed for doid: {}, error: {}", doid, e.getMessage()); + e.printStackTrace(); + return false; + } + } + + /** + * 删除LRS统计信息 + * @param doid 数据对象标识符 + * @return 操作是否成功 + */ + static public boolean deleteLRSStatistics(String doid) { + try { + byte[] doidBytes = doid.getBytes(StandardCharsets.UTF_8); + db.delete(lrsStatisticsCF, doidBytes); + return true; + } catch (RocksDBException e) { + LOGGER.error("[DAO ERROR] deleteLRSStatistics failed for doid: {}, error: {}", doid, e.getMessage()); + e.printStackTrace(); + return false; + } + } + + /** + * 更新LRS统计信息(用于updateLRS操作中doId更新的情况) + * @param oldDoid 旧的数据对象标识符 + * @param newDoid 新的数据对象标识符 + * @return 操作是否成功 + */ + static public boolean updateLRSStatisticsFromUpdateLRS(String oldDoid, String newDoid) { + try { + // 获取旧的统计信息 + String statisticsData = getLRSStatistics(oldDoid); + if (statisticsData == null) { + LOGGER.warn("[updateLRSStatisticsFromUpdateLRS] Cannot find LRSStatistics for oldDoid: {}, newDoid: {}", oldDoid, newDoid); + return false; + } + + // 使用批量操作确保原子性 + try (WriteBatch batch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { + byte[] newDoidBytes = newDoid.getBytes(StandardCharsets.UTF_8); + byte[] oldDoidBytes = oldDoid.getBytes(StandardCharsets.UTF_8); + byte[] dataBytes = statisticsData.getBytes(StandardCharsets.UTF_8); + + // 添加新记录 + batch.put(lrsStatisticsCF, newDoidBytes, dataBytes); + // 删除旧记录 + batch.delete(lrsStatisticsCF, oldDoidBytes); + + // 原子性提交 + db.write(writeOptions, batch); + return true; + } + } catch (Exception e) { + LOGGER.error("[DAO ERROR] updateLRSStatisticsFromUpdateLRS failed for oldDoid: {}, newDoid: {}, error: {}", oldDoid, newDoid, e.getMessage()); + e.printStackTrace(); + return false; + } + } + + /** + * 关闭数据库连接和释放资源 + */ + public static void close() { + synchronized (initLock) { + try { + if (db != null) { + // 关闭所有 ColumnFamily 句柄 + if (defaultCF != null) defaultCF.close(); + if (doidCF != null) doidCF.close(); + if (prefixCF != null) prefixCF.close(); + if (repoCF != null) repoCF.close(); + if (countCF != null) countCF.close(); + if (indexTimeIdCF != null) indexTimeIdCF.close(); + if (indexTimePrefixCF != null) indexTimePrefixCF.close(); + if (indexTimeRepoCF != null) indexTimeRepoCF.close(); + if (indexTimeCodeCF != null) indexTimeCodeCF.close(); + if (namePubkeyVerifyCF != null) namePubkeyVerifyCF.close(); + if (lrsStatisticsCF != null) lrsStatisticsCF.close(); + + // 关闭数据库 + db.close(); + db = null; + + // 重置初始化标志,允许重新初始化 + initialized = false; + LOGGER.info("DoidDAO closed successfully"); + } + } catch (Exception e) { + LOGGER.error("Error closing DoidDAO: {}", e.getMessage(), e); + e.printStackTrace(); + } + } + } + + /** + * 获取数据库统计信息 + */ + public static String getDBStats() { + try { + if (db != null) { + return db.getProperty("rocksdb.stats"); + } + } catch (RocksDBException e) { + e.printStackTrace(); + } + return "Database not available"; + } +} diff --git a/src/main/java/org/bdware/datanet/router/Global.java b/src/main/java/org/bdware/datanet/router/Global.java index 9fe6435574ab87659241880b3a7218287d929e99..fa8086009f9ec3eaebd3059893bd612ae2a2015e 100644 --- a/src/main/java/org/bdware/datanet/router/Global.java +++ b/src/main/java/org/bdware/datanet/router/Global.java @@ -1,8 +1,6 @@ package org.bdware.datanet.router; -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import com.google.gson.*; import io.netty.channel.*; import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; @@ -48,17 +46,23 @@ public class Global { // key = suffix of doid; value = doid; - public static RocksDBUtil lRSStorage = RocksDBUtil.loadDB("./lrsTable", false); +// public static RocksDBUtil lRSStorage = RocksDBUtil.loadDB("./lrsTable", false); // key = suffix - public static RocksDBUtil repoStorage = RocksDBUtil.loadDB("./repoStorage", false); +// public static RocksDBUtil repoStorage = RocksDBUtil.loadDB("./repoStorage", false); // key = suffix of doid; value = doid; - public static RocksDBUtil doStorage = RocksDBUtil.loadDB("./doStorage", false); +// public static RocksDBUtil doStorage = RocksDBUtil.loadDB("./doStorage", false); + + // key = suffix of doid; value = datadoidcnt 、 prefixcnt; +// public static RocksDBUtil lrsStatisticsStorage = RocksDBUtil.loadDB("./lrsStatisticsStorage", false); + + // key = pubkey, value = permission; 'ac' stands for ABAC (Attribute-Based Access Control), which prefers permissions from lower nodes +// public static RocksDBUtil acStorage = RocksDBUtil.loadDB("./acTable", false); // key = timestamp; value = doid - public static RocksDBUtil doidIndex = RocksDBUtil.loadDB("./doIndex", "doIndex"); - public static RocksDBUtil indexStorage = RocksDBUtil.loadDB("./indexStorage", false); +// public static RocksDBUtil doidIndex = RocksDBUtil.loadDB("./doIndex", "doIndex"); +// public static RocksDBUtil indexStorage = RocksDBUtil.loadDB("./indexStorage", false); public static RocksDBUtil otherURIRouterStorage = RocksDBUtil.loadDB("./otherURIRouterStorage", false); public static RocksDBUtil repoProxy = RocksDBUtil.loadDB("./repoProxy", false); @@ -70,15 +74,11 @@ public class Global { public static void resetEnvironment(Object arg) { try { + // 优雅地初始化DoidDAO + DoidDAO.initialize(); JsonObject jsonObject = JSONTool.convertMirrorToJson(arg).getAsJsonObject(); boolean isRoot = jsonObject.has("isRoot") && jsonObject.get("isRoot").getAsBoolean(); String routerUrl; - // fix: setEnableBindRepo 在得到 false 时,设置了 enableFlex 为 false - // if (jsonObject.has("enableBindRepo") - // && jsonObject.get("enableBindRepo").getAsBoolean()) { - // Global.config.setEnableBindRepo(true); - // } else - // Global.config.setEnableFlex(false); Global.config.setEnableFlex( jsonObject.has("enableFlex") && jsonObject.get("enableFlex").getAsBoolean()); Global.config.setEnableBindRepo(jsonObject.has("enableBindRepo") @@ -116,7 +116,7 @@ public class Global { for (int i = 0; i < createDefaultRepoNameList.length; i++) { String name = createDefaultRepoNameList[i]; String key = createDefaultRepoKeyList[i]; - String content = Global.doStorage.get(config.getPrefix() + "/" + name); + String content = DoidDAO.getDO(config.getPrefix() + "/" + name); if (content == null) { createDefaultRepo(jsonObject, key, name); } @@ -132,6 +132,7 @@ public class Global { } } catch (Exception e) { e.printStackTrace(); + LOGGER.error("Error when init environment: {}",e.getMessage()); } } @@ -147,6 +148,8 @@ public class Global { routeInfo.addProperty("protocol", "IRP"); routeInfo.addProperty("status", "online"); routeInfo.add("pubKey", jsonObject.get("publicKey")); + config.setPubKey(jsonObject.get("publicKey").getAsString()); + config.setPrivKey(jsonObject.get("privateKey").getAsString()); JsonObject wrapRouterInfo = new JsonObject(); wrapRouterInfo.addProperty("routerInfo", routeInfo.toString()); Global.config.setRootRouterInfo(wrapRouterInfo); @@ -159,6 +162,7 @@ public class Global { config.setUpperIP(jsonObject.get("routerIP").getAsString()); config.setUpperPort(jsonObject.get("routerPort").getAsString()); config.setPubKey(publicKey); + config.setPrivKey(privateKey); config.setName(name); SM2KeyPair keyPair = SM2KeyPair.fromJson(jsonObject.toString()); String signature = null; @@ -173,7 +177,7 @@ public class Global { } - private static void createDefaultRepo(JsonObject jsonObject, String key, String name) { + private static void createDefaultRepo(JsonObject jsonObject, String key, String name) throws Exception{ if (jsonObject.has(key) && !StringUtils.isEmpty(jsonObject.get(key).getAsString())) { JsonObject auditRepo = new JsonObject(); auditRepo.addProperty("date", System.currentTimeMillis()); @@ -182,9 +186,25 @@ public class Global { auditRepo.addProperty("address", jsonObject.get(key).getAsString()); auditRepo.addProperty("status", "online"); auditRepo.addProperty("protocol", "DOIP"); + auditRepo.addProperty("type",findType(name)); auditRepo.add("pubKey", jsonObject.get("publicKey")); auditRepo.addProperty("version", "2.1"); - Router.createInternal(auditRepo, Global.repoStorage); +// Router.createInternal(auditRepo, Global.repoStorage); + DoidDAO.putDO(auditRepo.get("doId").getAsString(), auditRepo); + } + } + + private static String findType(String name){ + if (name.contains("Repository")){ + return "system/bdware/repository"; + }else if (name.contains("Registry")){ + return "system/bdware/registry"; + }else if (name.contains(AutoAuditDO.AUDIT_PROXY)){ + return "system/bdware/audit-proxy"; + }else if (name.contains("AuditRepo")){ + return "system/bdware/audit-proxy"; + }else { + return "system/unknow"; } } @@ -197,8 +217,10 @@ public class Global { isVerified = false; verifyUpperRouter(); isVerified = true; + InfoSendTicker.init(); StatusChecker.init(); } catch (Exception e) { + InfoSendTicker.init(); StatusChecker.init(); isVerified = false; e.printStackTrace(); @@ -229,15 +251,15 @@ public class Global { } public static JsonObject listAllIndexStorage() { - RocksIterator iter = Global.indexStorage.newIterator(); - iter.seekToFirst(); - JsonObject ret = new JsonObject(); - for (; iter.isValid(); iter.next()) { - String key = new String(iter.key()); - String value = new String(iter.value()); - ret.addProperty(key, value); - } - return ret; +// RocksIterator iter = Global.indexStorage.newIterator(); +// iter.seekToFirst(); +// JsonObject ret = new JsonObject(); +// for (; iter.isValid(); iter.next()) { +// String key = new String(iter.key()); +// String value = new String(iter.value()); +// ret.addProperty(key, value); +// } + return DoidDAO.listNamePubkey(); } static public void verifyUpperRouter() throws IrpClientException { @@ -366,8 +388,10 @@ public class Global { List listenerInfos = new ArrayList(); listenerInfos.add(listenerInfo); IrpServerInfo info = new IrpServerInfo(config.getPrefix(), "no desc", listenerInfos); - String auditProxy = - Global.doStorage.get(AutoAuditDO.getAuditProxyDoid(config.getPrefix())); +// String auditProxy = +// Global.doStorage.get(AutoAuditDO.getAuditProxyDoid(config.getPrefix())); + + String auditProxy = DoidDAO.getDO(AutoAuditDO.getAuditProxyDoid(config.getPrefix())); AuditConfig auditConfig = null; if (auditProxy != null) { JsonObject jo = JsonParser.parseString(auditProxy).getAsJsonObject(); @@ -387,7 +411,10 @@ public class Global { }; } else { String auditProxyDoid = AutoAuditDO.getAuditProxyDoid(Global.config.getPrefix()); - String auditProxyInfo = Global.repoStorage.get(auditProxyDoid); + String auditProxyInfo = DoidDAO.getDO(auditProxyDoid); + if (auditProxyInfo == null) { + LOGGER.error("audit proxy not found! use NoneAudit"); + } String address = JsonParser.parseString(auditProxyInfo).getAsJsonObject() .get("address").getAsString(); auditConfig = new NoneAuditConfig(new AuditRepo(auditProxyDoid, address)); @@ -421,7 +448,7 @@ public class Global { } public static String getRepo(String repoId) { - return Global.repoStorage.get(repoId); + return DoidDAO.getDO(repoId); } public static JsonObject peekDB(RocksDBUtil table) { diff --git a/src/main/java/org/bdware/datanet/router/IDFilter.java b/src/main/java/org/bdware/datanet/router/IDFilter.java index d91db690e302c68d658c9370d92b42fd5a1b4746..49d4f7f89f775301b8db96f325f498f752555c9d 100644 --- a/src/main/java/org/bdware/datanet/router/IDFilter.java +++ b/src/main/java/org/bdware/datanet/router/IDFilter.java @@ -35,22 +35,22 @@ public abstract class IDFilter { case dataId: return new IDFilter() { @Override - public boolean accept(String id) { - return countSubstrings(id, "/") > 1; + public boolean accept(String type) { + return !type.startsWith("system") && !type.startsWith("prefix") ; } }; case prefixId: return new IDFilter() { @Override - public boolean accept(String id) { - return countSubstrings(id, "/") == 0; + public boolean accept(String type) { + return type.equalsIgnoreCase("prefix"); } }; case repoId: return new IDFilter() { @Override - public boolean accept(String id) { - return countSubstrings(id, "/") == 1; + public boolean accept(String type) { + return type.startsWith("system"); } }; default: diff --git a/src/main/java/org/bdware/datanet/router/InfoSendTicker.java b/src/main/java/org/bdware/datanet/router/InfoSendTicker.java new file mode 100644 index 0000000000000000000000000000000000000000..990327ff25dd01f20393bd13f2100621113d5aa9 --- /dev/null +++ b/src/main/java/org/bdware/datanet/router/InfoSendTicker.java @@ -0,0 +1,99 @@ +package org.bdware.datanet.router; + +import com.google.gson.JsonObject; +import org.bdware.client.ContractRequest; +import org.bdware.ypkdeploy.SmartContractClientExt; +import org.zz.gmhelper.SM2KeyPair; + +import java.util.concurrent.*; + +import static org.bdware.datanet.router.RowStore.LOGGER; + +public class InfoSendTicker { + // 单线程守护调度器 + private static final ScheduledExecutorService SCHEDULER = + Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "InfoSendTicker"); + t.setDaemon(true); + return t; + }); + private static volatile ScheduledFuture FUTURE; + + // 每次 snapshot 的快照数据 + private final int prefixCnt; + private final int dataDoidCnt; + + private InfoSendTicker(int prefix, int dataDoid) { + this.prefixCnt = prefix; + this.dataDoidCnt = dataDoid; + } + + public int getPrefixCnt() { return prefixCnt; } + public int getDataDoidCnt() { return dataDoidCnt; } + + //================================================ + // 1. 直接从 DoidDAO 读取计数,创建快照 + //================================================ + public static InfoSendTicker createSnapshot() { + int prefixCount = DoidDAO.getCount("prefix"); + int dataCount = DoidDAO.getCount("data"); + + return new InfoSendTicker(prefixCount, dataCount); + } + + //================================================ + // 2. 定时发送:每 2 小时从表中读取计数并发给上级 + //================================================ + public static synchronized void init() { + if (FUTURE != null && !FUTURE.isDone()) { + return; + } + + // 先读取一次当前计数,记录初始状态 + InfoSendTicker initialSnapshot = createSnapshot(); + LOGGER.info("Initial counts: prefix={}, dataDoid={}", + initialSnapshot.getPrefixCnt(), initialSnapshot.getDataDoidCnt()); + + FUTURE = SCHEDULER.scheduleWithFixedDelay(() -> { + InfoSendTicker snap = createSnapshot(); + + LOGGER.info("Snapshot collected: prefix={}, dataDoid={}", + snap.getPrefixCnt(), snap.getDataDoidCnt()); + try { + sendToUpperNode(snap); + } catch (Exception e) { + LOGGER.error("Failed to send to upper node", e); + } + }, 100, 7200, TimeUnit.SECONDS); + + LOGGER.info("InfoSendTicker scheduled every 2 hours"); + } + + /** 优雅停止定时任务 */ + public static synchronized void shutdown() { + if (FUTURE != null) { + FUTURE.cancel(false); + } + SCHEDULER.shutdownNow(); + } + + private static void sendToUpperNode(InfoSendTicker snap) throws Exception { + String targetUrl = "ws://" + Global.config.getUpperIP() + ":21030" + "/SCIDE/SCExecutor"; + JsonObject body = new JsonObject(); + body.addProperty("doid", Global.config.getPrefix()); + body.addProperty("prefixcnt", snap.getPrefixCnt()); + body.addProperty("datadoidcnt", snap.getDataDoidCnt()); + + JsonObject keyPair = new JsonObject(); + keyPair.addProperty("publicKey", Global.config.getPubKey()); + keyPair.addProperty("privateKey", Global.config.getPrivKey()); + SmartContractClientExt client = new SmartContractClientExt(targetUrl, SM2KeyPair.fromJson(keyPair.toString())); + client.waitForConnect(); + + ContractRequest cr = new ContractRequest(); + cr.setArg( body); + cr.setAction("storageLRSStatistics"); + cr.setContractID("GlobalRouter"); + client.executeContractSync(cr); + } +} \ No newline at end of file diff --git a/src/main/java/org/bdware/datanet/router/ReadMe.md b/src/main/java/org/bdware/datanet/router/ReadMe.md new file mode 100644 index 0000000000000000000000000000000000000000..280ad8865add5c0fe716d4d1242b8e324bb38677 --- /dev/null +++ b/src/main/java/org/bdware/datanet/router/ReadMe.md @@ -0,0 +1,42 @@ +## 开发文档 + +### permission 分析 +RBAC.yjs 指定了通过方法调用的角色以身份为主体(节点与节点之间通信),这是为了保证子节点可以有权限调用父节点的方法。 + +RBAC模型是一种好的实践方案。但是这个服务无法在一个用户添加前后缀的时候通过合约调用(端口占用)。 + +在代码中,acTable指向的是各个公钥的权限,所以在增删改前缀的时候需要充分考虑公钥可能出现的情况。 + +* 简单来说,添加前缀时直接把用户传递的公钥添加权限。 删除前缀时如果这个公钥还只向了别的前缀时,就保留,否则删除此公钥的权限。 + +* 修改前缀时,由于前端未传递修改之前的公钥,所以需要通过oldDoid去获取oldPubkey,比对二者。 + +* 如果不同,那么就对新公钥添加权限,但是原公钥可能还指向别的公钥,所以需要判断是否可被删除权限。 + +这种设计方案有什么问题呢? +* 如果一位用户的原公钥泄露,恰巧有其他前缀包含了原公钥导致权限无法删除,那么攻击者就可使用原公钥的权限伪造行为。 + +解决方案:不使用公钥指代角色,或者使用公钥+前缀组合细化角色,隔离权限 + +![img.png](../../../../../resources/imgs/权限-主体分析.png) + + + +![img.png](../../../../../resources/imgs/权限-问题.png) + + + +![img.png](../../../../../resources/imgs/权限-攻击.png) + + + +### LRSStatistics 方法分析 +该方法一共有三个,分别是get、store、 delete +* 方法的作用是子节点向父节点发送自己的前缀数量和dataDoid数量,系统启动时执行一次,后续每两小时执行一次。 +* 子节点向父节点调用方法需要有权限,具体可参考上述的`authorizedChildren`。 +* 数量是原子操作修改的,具体参考[InfoSendTicker.java](InfoSendTicker.java),调用的位置分别在: LRS(create、delete),removeDoid、putDO +* 父节点收到请求后会将其写入自身的`statistics`表中。`key`是子节点传递的`doid`,`doid`获取自**子节点**系统启动的`prefix`参数,具体可参考`cmconfig.json`. +* 定时推送的逻辑会在判断非根节点后执行。[Global.java](Global.java) :Global.verifyAndStartServer, 一般来说isRoot表示节点是否为根节点,但节点是否是父节点应该参考配置文件中的`routerAddress`字段,表示上层的通信地址。 +* 具体推送的时候获取的`upperIP`即是父节点的ip地址,但是合约跑在21030端口,并且协议是ws,因此需要手动拼接地址。 +* 配置文件中path需要指向子节点模式的ypk文件,所以需要单独build,`routerAddress` 、 `isRoot` 都需要对应修改。 +* 如果能推送成功,**父节点必须有子节点的前缀信息**,具体是`publicKey`+`doId` \ No newline at end of file diff --git a/src/main/java/org/bdware/datanet/router/Router.java b/src/main/java/org/bdware/datanet/router/Router.java index 77f807dbfac06c86c27d278e56221b5b125ebc29..fed5b3dc270c954118c561082ab4ce004d013300 100644 --- a/src/main/java/org/bdware/datanet/router/Router.java +++ b/src/main/java/org/bdware/datanet/router/Router.java @@ -4,6 +4,7 @@ import com.google.gson.*; import io.netty.util.internal.StringUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.bdware.client.ContractRequest; import org.bdware.datanet.router.irp.CommonServerHandler; import org.bdware.datanet.router.irp.IrsHandlerImpl; import org.bdware.doip.encrypt.SM2Signer; @@ -14,6 +15,7 @@ import org.bdware.sc.boundry.JavaScriptEntry; import org.bdware.sc.boundry.utils.RocksDBUtil; import org.bdware.sc.engine.JSONTool; import org.bdware.sc.util.JsonUtil; +import org.bdware.ypkdeploy.SmartContractClientExt; import org.rocksdb.RocksIterator; import org.zz.gmhelper.SM2KeyPair; import wrp.jdk.nashorn.api.scripting.ScriptObjectMirror; @@ -22,33 +24,17 @@ import wrp.jdk.nashorn.internal.scripts.JO; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; + public class Router { private static final Logger LOGGER = LogManager.getLogger(Router.class); public static Object clear() { - JsonObject ret = new JsonObject(); - RocksDBUtil[] db = {Global.lRSStorage, Global.repoStorage, Global.indexStorage}; - RocksIterator[] iter = {Global.lRSStorage.newIterator(), Global.repoStorage.newIterator(), - Global.indexStorage.newIterator()}; - ret.addProperty("result", "success"); - try { - for (int i = 0; i < iter.length; i++) { - for (iter[i].seekToFirst(); iter[i].isValid(); iter[i].next()) { - String jsonStr = new String(iter[i].key()); - if (db[i].delete(jsonStr) == "failed") { - ret.addProperty("result", "failed"); - } - } - } - } catch (Exception e) { - ret.addProperty("result", "failed"); - ByteArrayOutputStream bo = new ByteArrayOutputStream(); - e.printStackTrace(new PrintStream(bo)); - ret.addProperty("errorMessage", bo.toString()); - } - return ret; + return DoidDAO.clearAllStorage(); } public static Object clearConfig() { @@ -82,68 +68,332 @@ public class Router { } public static Object clearDoStorage() { + return DoidDAO.clearAllStorage(); + } + + public static Object listLRS(ScriptObjectMirror obj) { + JsonArray ja = new JsonArray(); JsonObject ret = new JsonObject(); - RocksDBUtil[] db = {Global.doStorage}; - RocksIterator[] iter = {Global.doStorage.newIterator()}; - ret.addProperty("result", "success"); + JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + int offset = jo.has("offset") ? jo.get("offset").getAsInt() : 0; + int count = jo.has("count") ? jo.get("count").getAsInt() : 10; + String doid = jo.has("doId") ? jo.get("doId").getAsString(): null; + Long startTs = jo.has("startTs")?jo.get("startTs").getAsLong():null; + Long endTs = jo.has("endTs")?jo.get("endTs").getAsLong():null; + boolean ascending = !("desc".equalsIgnoreCase(jo.has("order")? jo.get("order").getAsString(): "")); + try { - for (int i = 0; i < iter.length; i++) { - for (iter[i].seekToFirst(); iter[i].isValid(); iter[i].next()) { - String jsonStr = new String(iter[i].key()); - if (db[i].delete(jsonStr) == "failed") { - ret.addProperty("result", "failed"); - } - } + JsonObject res ; + if (doid == null) { + res = IrsHandlerImpl.queryLRSByOffsetByRocksDB(startTs,endTs,offset, count, ascending); + } else{ + // doid 匹配,以前缀开头进行模糊搜索 + res = IrsHandlerImpl.queryLRSByDoid(doid,offset, count, ascending); } - } catch (Exception e) { - ret.addProperty("result", "failed"); - ByteArrayOutputStream bo = new ByteArrayOutputStream(); - e.printStackTrace(new PrintStream(bo)); - ret.addProperty("errorMessage", bo.toString()); + for (String id : res.keySet()){ + JsonObject doResult = res.get( id).getAsJsonObject(); + doResult.addProperty("doId", id); + ja.add(doResult); + } + ret.addProperty("status", "success"); + ret.addProperty("total", DoidDAO.getCount("prefix")); + ret.addProperty("count", ja.size()); + ret.add("data", ja); + + }catch (Exception e){ + e.printStackTrace(); + ret.addProperty("status", "error"); + ret.addProperty("errorMessage", e.getMessage()); } - return ret; - } - public static Object listLRS() { - return listValues(Global.lRSStorage); + return ret; } public static Object createLRS(ScriptObjectMirror obj) { - return create(obj, Global.lRSStorage); + obj.setMember("type", "prefix"); + updateChildPermission(obj); + return create(obj); } public static Object updateLRS(ScriptObjectMirror obj) { - return update(obj, Global.lRSStorage); + obj.setMember("type", "prefix"); + updateChildPermission(obj); + + + // 检查是否有doId更新,如果有则需要更新LRSStatistics + JsonObject jo = JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + if (jo.has("oldDoId") && jo.has("doId")) { + String oldDoId = jo.get("oldDoId").getAsString(); + String newDoId = jo.get("doId").getAsString(); + if (!oldDoId.equals(newDoId)) { + updateLRSStatisticsFromUpdateLRS(oldDoId, newDoId); + } + } + + return update(obj); } public static Object deleteLRS(ScriptObjectMirror obj) { - return delete(obj, Global.lRSStorage); + deleteChildPermission(obj); + return delete(obj); } public static Object getLRS(ScriptObjectMirror obj) { - return get(obj, Global.lRSStorage); + return get(obj); + } + + // 注:添加权限的操作是在 LRS 发生增添、修改、删除之后直接添加 authorizedChildren 权限, + // 保证LRSStatistics的方法能够正常调用(子节点调用主节点) + public static void updateChildPermission(ScriptObjectMirror obj) { + JsonObject jo = JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + if (!jo.has("pubKey") ) { + LOGGER.warn("[updateChildPermission] invalid params, missing required: pubKey"); + return; + } + + String pubkey = jo.get("pubKey").getAsString(); + String oldDoid = jo.has("oldDoId") ? jo.get("oldDoId").getAsString() : null; + + // 异步调用智能合约添加角色权限(新pubkey) + executeSmartContractRole(pubkey, "addRole"); + + // 处理旧权限清理 + if (oldDoid != null) { + handleOldDoidPermissions(oldDoid, pubkey); + } + } + + /** + * 执行智能合约角色操作 + */ + private static void executeSmartContractRole(String pubkey, String action) { + CompletableFuture.runAsync(() -> { + try { + String targetUrl = "ws://" + Global.config.getUpperIP() + ":21030" + "/SCIDE/SCExecutor"; + JsonObject body = new JsonObject(); + body.addProperty("user", pubkey); + body.addProperty("role", "authorizedChildren"); + + JsonObject keyPair = new JsonObject(); + keyPair.addProperty("publicKey", Global.config.getPubKey()); + keyPair.addProperty("privateKey", Global.config.getPrivKey()); + + SmartContractClientExt client = new SmartContractClientExt(targetUrl, SM2KeyPair.fromJson(keyPair.toString())); + client.waitForConnect(); + + ContractRequest cr = new ContractRequest(); + cr.setArg(body); + cr.setAction(action); + cr.setContractID("GlobalRouter"); + client.executeContract(cr); + + LOGGER.info("[executeSmartContractRole] {} executed successfully for pubkey: {}", action, pubkey); + } catch (Exception e) { + LOGGER.error("[executeSmartContractRole] Failed to execute {} for pubkey: {}, error: {}", action, pubkey, e.getMessage()); + } + }); + } + + + + /** + * 处理oldDoid的权限清理 + */ + private static void handleOldDoidPermissions(String oldDoid, String newPubkey) { + if (!DoidDAO.isExistedDoid(oldDoid)) { + LOGGER.warn("[handleOldDoidPermissions] Old DOID {} does not exist in storage", oldDoid); + return; + } + + // 从数据库中查出oldDoid对应的pubkey + JsonObject oldDoidData = JsonParser.parseString(DoidDAO.getDO(oldDoid)).getAsJsonObject(); + String oldPubkey = oldDoidData.get("pubKey").getAsString(); + + // 比对oldPubkey和newPubkey是否相同,如果相同就不用管了 + if (Objects.equals(oldPubkey, newPubkey)) { + return; + } + + // 如果不同,则尝试删除oldPubkey + removePubkeyPermission(oldPubkey, oldDoid); + } + + /** + * 删除公钥权限 + */ + private static void removePubkeyPermission(String pubkey, String excludeDoid) { + // 检查公钥是否仍被其他前缀使用 + if (DoidDAO.isPubKeyUsedByOtherPrefix(pubkey, excludeDoid)) { + LOGGER.info("[removePubkeyPermission] PubKey {} still in use by other prefixes, skipping removal", pubkey); + return; + } + + // 异步调用智能合约删除角色权限 + executeSmartContractRole(pubkey, "removeRole"); + LOGGER.info("[removePubkeyPermission] Permission removal initiated for pubKey: {}", pubkey); + } + + public static void deleteChildPermission(ScriptObjectMirror obj){ + JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + if (!jo.has("doId")) { + LOGGER.warn("[deleteChildPermission] invalid params, missing: doId"); + return; + } + + String doid = jo.get("doId").getAsString(); + + if (!DoidDAO.isExistedDoid(doid)) { + LOGGER.warn("[deleteChildPermission] DOID {} does not exist in storage", doid); + return; + } + + // 从数据库中查出doid对应的pubkey,然后尝试删除 + JsonObject doidData = JsonParser.parseString(DoidDAO.getDO(doid)).getAsJsonObject(); + String pubKey = doidData.get("pubKey").getAsString(); + + removePubkeyPermission(pubKey, doid); + } + + public static Object getLRSStatistics(ScriptObjectMirror obj){ + JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + JsonObject ret = new JsonObject(); + + if (!jo.has("doid")){ + ret.addProperty("result", "invalid params"); + return ret; + } + String res = DoidDAO.getLRSStatistics(jo.get("doid").getAsString()); + if (null != res){ + ret.addProperty("result", "success"); + ret.add("data", JsonParser.parseString(res)); + }else{ + ret.addProperty("result", "failed"); + ret.addProperty("message", "no data"); + }; + return ret; + } + + // 参数可传可不传 + public static Object getLRSStatisticsList(ScriptObjectMirror obj){ + JsonObject ret = new JsonObject(); + try { + List statisticsList = DoidDAO.getLRSStatisticsList(); + JsonArray dataArray = new JsonArray(); + for (String statisticsData : statisticsList) { + dataArray.add(JsonParser.parseString(statisticsData)); + } + ret.addProperty("result", "success"); + ret.add("data", dataArray); + } catch (Exception e) { + LOGGER.error("[getLRSStatisticsList] error: {}", e.getMessage()); + ret.addProperty("result", "failed"); + ret.addProperty("errorMessage", e.getMessage()); + } + return ret; } - public static Object listRepo() { - return listValues(Global.repoStorage); + public static Object storageLRSStatistics(ScriptObjectMirror obj){ + JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + JsonObject ret = new JsonObject(); + if (!(jo.has("doid") && jo.has("prefixcnt") && jo.has("datadoidcnt"))){ + ret.addProperty("result", "invalid params"); + return ret; + } + String doid = jo.get("doid").getAsString(); + // 判断doid在前缀列表中是否存在. + if (DoidDAO.getDO(doid) == null){ + ret.addProperty("result", "failed"); + ret.addProperty("message", "lrs is not exists"); + return ret; + } + boolean success = DoidDAO.storageLRSStatistics(doid, jo.toString()); + if (success) { + ret.addProperty("result", "success"); + } else { + ret.addProperty("result", "failed"); + ret.addProperty("message", "storage operation failed"); + } + return ret; + } + + public static void updateLRSStatisticsFromUpdateLRS(String oldDoid, String newDoid){ + boolean success = DoidDAO.updateLRSStatisticsFromUpdateLRS(oldDoid, newDoid); + if (!success) { + LOGGER.warn("[updateLRSStatisticsFromUpdateLRS] Failed to update LRSStatistics from oldDoid: {} to newDoid: {}", oldDoid, newDoid); + } + } + + public static Object deleteLRSStatistics(ScriptObjectMirror obj) { + JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + JsonObject ret = new JsonObject(); + if (!jo.has("doid")) { + ret.addProperty("result", "invalid params"); + return ret; + } + boolean success = DoidDAO.deleteLRSStatistics(jo.get("doid").getAsString()); + if (success) { + ret.addProperty("result", "success"); + } else { + ret.addProperty("result", "failed"); + } + return ret; + } + + public static Object listRepo(ScriptObjectMirror obj) { + JsonObject ret = new JsonObject(); + JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + int offset = jo.has("offset") ? jo.get("offset").getAsInt() : 0; + int count = jo.has("count") ? jo.get("count").getAsInt() : 10; + String doid = jo.has("doId") ? jo.get("doId").getAsString(): null; + String type = jo.has("type") ? jo.get("type").getAsString(): null; + Long startTs = jo.has("startTs")?jo.get("startTs").getAsLong():null; + Long endTs = jo.has("endTs")?jo.get("endTs").getAsLong():null; + boolean ascending = !("desc".equalsIgnoreCase(jo.has("order") ? jo.get("order").getAsString():"")); + try { + JsonObject res; + if (doid == null){ + res = IrsHandlerImpl.queryRepositoryByOffsetByRocksDB(type,startTs,endTs,offset, count,ascending); + }else{ + res = IrsHandlerImpl.queryRepositoryByDoid(doid,offset, count,ascending); + } + JsonArray ja = new JsonArray(); + for (String id : res.keySet()) { + JsonObject doResult = res.getAsJsonObject(id); + doResult.addProperty("doId", id); + ja.add(doResult); + } + ret.addProperty("result", "success"); + ret.add("data", ja); + ret.addProperty("total", DoidDAO.getCount("system")); + ret.addProperty("count", ja.size()); + } catch (Exception e){ + LOGGER.error("[listRepo] error: {}", e.getMessage()); + ret.addProperty("result", "failed"); + ret.addProperty("errorMessage", e.toString()); + } + return ret; } public static Object createRepo(ScriptObjectMirror obj) { if (!obj.hasMember("status")) obj.setMember("status", "online"); - return create(obj, Global.repoStorage); + if (!obj.hasMember("type")){ + obj.setMember("type", "system/unknown"); + } + return create(obj); } public static Object updateRepo(ScriptObjectMirror obj) { - return update(obj, Global.repoStorage); + // I think there should be a better way to get the "type" of a repoId, or get it from frontend. + return update(obj); } public static Object deleteRepo(ScriptObjectMirror obj) { - return delete(obj, Global.repoStorage); + return delete(obj); } public static Object getRepo(ScriptObjectMirror obj) { - return get(obj, Global.repoStorage); + return get(obj); } public static Object setPubKey(ScriptObjectMirror obj) { @@ -292,25 +542,32 @@ public class Router { } // TODO design the jsonobject, doi={using full doi}? doi={ .xx}? - public static Object create(ScriptObjectMirror obj, RocksDBUtil table) { + public static Object create(ScriptObjectMirror obj) { JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); - return createInternal(jo, table); + return createInternal(jo); } - public static JsonObject createInternal(JsonObject jo, RocksDBUtil table) { + // prefixId type default: prefix + // repoId type default: system/unknown + public static JsonObject createInternal(JsonObject jo) { JsonObject ret = new JsonObject(); ret.addProperty("result", "success"); - if (jo.has("doId") && jo.has("name") && jo.has("pubKey")) { - String doId = jo.get("doId").getAsString(); - String name = jo.get("name").getAsString(); - String pubKey = jo.get("pubKey").getAsString(); - String indexKey = name + "|" + pubKey; - addAddressIntoOtherAddrs(jo); - if (!acceptCreate(Global.config.getPrefix(), doId)) { + if (jo.has("doId") && jo.has("name") && jo.has("pubKey") && jo.has("type")) { + try{ + String doId = jo.get("doId").getAsString(); + String name = jo.get("name").getAsString(); + String pubKey = jo.get("pubKey").getAsString(); + String indexKey = name + "|" + pubKey; + addAddressIntoOtherAddrs(jo); + if (!acceptCreate(Global.config.getPrefix(), doId)) { + ret.addProperty("result", "failed"); + ret.addProperty("data", "invalid doId or can not get current prefix"); + } else { + ret = doStoreAndIndex(ret, jo, doId, indexKey); + } + } catch (Exception e){ ret.addProperty("result", "failed"); - ret.addProperty("data", "invalid doId or can not get current prefix"); - } else { - ret = doStoreAndIndex(ret, jo, table, doId, indexKey); + LOGGER.error("[createInternal error: ]" + e.getMessage()); } } else { ret.addProperty("result", "failed"); @@ -329,6 +586,7 @@ public class Router { array.add(val); } } catch (Exception e) { + LOGGER.error("[addAddressIntoOtherAddrs error: ]" + e.getMessage()); } } @@ -350,7 +608,7 @@ public class Router { return false; } - public static Object update(ScriptObjectMirror obj, RocksDBUtil table) { + public static Object update(ScriptObjectMirror obj) { JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); JsonObject ret = new JsonObject(); if (jo.has("oldDoId") && jo.has("oldName") && jo.has("pubKey") && jo.has("doId") @@ -362,23 +620,23 @@ public class Router { String oldName = jo.get("oldName").getAsString(); String indexKey = name + "|" + pubKey; String oldIndexKey = oldName + "|" + pubKey; - if (Global.config.getPrefix() == "") { + if (Objects.equals(Global.config.getPrefix(), "")) { ret.addProperty("result", "failed"); ret.addProperty("data", "invalid doId or can not get current prefix"); } else { - if (table.get(oldDoId) == null || table.delete(oldDoId) == "failed") { + if (!DoidDAO.isExistedDoid(oldDoId)){ ret.addProperty("result", "failed"); ret.addProperty("data", "can not find"); } else { - Global.doStorage.delete(oldDoId); - if (jo.has("pubKey")) { - if (Global.indexStorage.get(oldIndexKey) != null) { - Global.indexStorage.delete(oldIndexKey); - } - } + DoidDAO.deleteDO(oldDoId); + DoidDAO.deleteNamePubkey(oldIndexKey); jo.remove("oldDoId"); jo.remove("oldName"); - ret = doStoreAndIndex(ret, jo, table, doId, indexKey); + try{ + ret = doStoreAndIndex(ret, jo, doId, indexKey); + } catch (Exception e) { + LOGGER.error("[update error: {}]" , e.getMessage()); + } } } } else { @@ -389,7 +647,7 @@ public class Router { return ret; } - public static Object delete(ScriptObjectMirror obj, RocksDBUtil table) { + public static Object delete(ScriptObjectMirror obj) { JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); JsonObject ret = new JsonObject(); if (jo.has("doId") && jo.has("name") && jo.has("pubKey")) { @@ -397,19 +655,21 @@ public class Router { String name = jo.get("name").getAsString(); String pubKey = jo.get("pubKey").getAsString(); String indexKey = name + "|" + pubKey; - if (Global.config.getPrefix() == "") { + if (Objects.equals(Global.config.getPrefix(), "")) { ret.addProperty("result", "failed"); ret.addProperty("data", "can not get current prefix"); } else { - if (table.get(doId) == null || table.delete(doId) == "failed") { + if (!DoidDAO.isExistedDoid(doId)) { ret.addProperty("result", "failed"); ret.addProperty("data", "can not find"); } else { - Global.doStorage.delete(doId); + // 这两行属实是无奈之举 + JsonObject j = JsonParser.parseString(Objects.requireNonNull(DoidDAO.getDO(doId))).getAsJsonObject(); + jo.addProperty("type", j.get("type").getAsString()); + DoidDAO.deleteDO(doId); + if (jo.has("pubKey")) { - if (Global.indexStorage.get(indexKey) != null) { - Global.indexStorage.delete(indexKey); - } + DoidDAO.deleteNamePubkey(indexKey); } ret.addProperty("result", "success"); ret.addProperty("data", doId); @@ -418,7 +678,6 @@ public class Router { } else { ret.addProperty("result", "failed"); ret.addProperty("data", "missing arguments"); - } return ret; } @@ -453,62 +712,41 @@ public class Router { public static Object listID(ScriptObjectMirror obj) { JsonObject ret; - try { - JsonObject jo = - org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); + JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); - int offset = 0, count = 10; - if (jo.has("offset")) { - offset = jo.get("offset").getAsInt(); - } - if (jo.has("count")) { - count = jo.get("count").getAsInt(); - } - IDFilter idFilter; - if (jo.has("filter")) { - idFilter = IDFilter - .fromConfig(IDFilter.FilterType.valueOf(jo.get("filter").getAsString())); - } else - idFilter = IDFilter.defaultFilter(); - boolean ascending = true; - if (jo.has("order")) { - ascending = !jo.get("order").getAsString().equals("desc"); - } - if (jo.has("queryByCreateTime") && jo.get("queryByCreateTime").getAsBoolean()) { - ret = IrsHandlerImpl.queryIdentifierByOffsetByTimeIndex(offset, count, idFilter, - ascending); - } else { - ret = IrsHandlerImpl.queryIdentifierByOffsetByRocksDB(offset, count, idFilter, - ascending); - } + int offset = jo.has("offset") ? jo.get("offset").getAsInt() : 0; + int count = jo.has("count") ? jo.get("count").getAsInt() : 10; + boolean ascending = !jo.has("order") || jo.get("order").getAsString().equals("asc"); + Long startTs = jo.has("startTs")?jo.get("startTs").getAsLong():null; + Long endTs = jo.has("endTs")?jo.get("endTs").getAsLong():null; + try { + JsonObject ja = IrsHandlerImpl.queryDataByOffsetByRocksDB(startTs,endTs,offset, count, ascending); JsonObject expand = new JsonObject(); - for (String key : ret.keySet()) { + for (String key : ja.keySet()) { if (key.equals("data")) { - JsonArray array = ret.get(key).getAsJsonArray(); + JsonObject array = ja.get(key).getAsJsonObject(); JsonArray expandArray = new JsonArray(); - for (JsonElement idAsJE : array) { - String id = idAsJE.getAsString(); - String value = Global.doStorage.get(id); - JsonObject doReuslt = JsonUtil.parseString(value).getAsJsonObject(); - doReuslt.addProperty("doId", id); - if (doReuslt.has("repoId")) { + for (String id : array.keySet()) { + JsonObject doResult = array.get(id).getAsJsonObject(); + doResult.addProperty("doId", id); + if (doResult.has("repoId")) { JsonObject repoInfo = CommonServerHandler - .getRepoInfo(doReuslt.get("repoId").getAsString(), id); + .getRepoInfo(doResult.get("repoId").getAsString(), id); if (repoInfo != null && repoInfo.has("address") && repoInfo.has("version")) { - if (!doReuslt.has("address")) - doReuslt.addProperty("address", + if (!doResult.has("address")) + doResult.addProperty("address", repoInfo.get("address").getAsString()); - if (!doReuslt.has("version")) - doReuslt.addProperty("version", + if (!doResult.has("version")) + doResult.addProperty("version", repoInfo.get("version").getAsString()); } } - expandArray.add(doReuslt); + expandArray.add(doResult); } expand.add(key, expandArray); } else { - expand.add(key, ret.get(key)); + expand.add(key, ja.get(key)); } } return expand; @@ -522,7 +760,7 @@ public class Router { } } - public static Object get(ScriptObjectMirror obj, RocksDBUtil table) { + public static Object get(ScriptObjectMirror obj) { JsonObject jo = org.bdware.sc.engine.JSONTool.convertMirrorToJson(obj).getAsJsonObject(); JsonObject ret = new JsonObject(); if (jo.has("name") && jo.has("pubKey") && jo.has("signature")) { @@ -533,13 +771,13 @@ public class Router { if (!Config.verify(pubKey, info, signature)) { ret.addProperty("result", "failed"); ret.addProperty("data", "can not verify the key"); - } else if (Global.indexStorage.get(info) == null) { + } else if (DoidDAO.getNamePubkey(info) == null) { ret.addProperty("result", "failed"); ret.addProperty("data", "can not find"); } else { - String doId = Global.indexStorage.get(info); + String doId = DoidDAO.getNamePubkey(info); ret.addProperty("result", "success"); - ret.addProperty("data", table.get(doId)); + ret.addProperty("data", DoidDAO.getDO(doId)); } } else { ret.addProperty("result", "failed"); @@ -564,19 +802,18 @@ public class Router { } - private static JsonObject doStoreAndIndex(JsonObject ret, JsonObject jo, RocksDBUtil table, - String doId, String indexKey) { - // if (Global.indexStorage.get(indexKey) != null) { - // ret.addProperty("result", "failed"); - // ret.addProperty("data", "duplicate"); - // return ret; - // } + private static JsonObject doStoreAndIndex(JsonObject ret, JsonObject jo, + String doId, String indexKey) throws Exception{ + if (DoidDAO.getNamePubkey(indexKey) != null) { + ret.addProperty("result", "failed"); + ret.addProperty("data", "duplicate"); + return ret; + } CommonServerHandler.cacheResolve.invalidate(doId); ret.addProperty("result", "success"); ret.addProperty("data", doId); - Global.indexStorage.put(indexKey, doId); - Global.doStorage.put(doId, jo.toString()); - table.put(doId, jo.toString()); + DoidDAO.putNamePubkey(indexKey, doId); + DoidDAO.putDO(doId, jo); return ret; } @@ -597,6 +834,8 @@ public class Router { ByteArrayOutputStream bo = new ByteArrayOutputStream(); e.printStackTrace(new PrintStream(bo)); ret.addProperty("errorMessage", bo.toString()); + }finally { + iter.close(); } return ret; } @@ -690,14 +929,13 @@ public class Router { + routerInfoJO.get("port").getAsInt(); IrpRouterClientImpl client = new IrpRouterClientImpl("clientTest", url, null); StateInfoBase stateInfo = client.resolve(doId); - LOGGER.info("resolve:" + JsonUtil.toPrettyJson(stateInfo)); if (stateInfo == null) { LOGGER.info("Resolve failed!"); return "{}"; } return stateInfo.handleValues.toString(); } catch (Exception e) { - e.printStackTrace(); + LOGGER.error("[resolveTest failed!] {}", e.getMessage()); } return "{}"; } @@ -721,7 +959,8 @@ public class Router { LOGGER.info("register:" + stateInfo); } catch (Exception e) { - e.printStackTrace(); + LOGGER.error("[registerTest failed!] {}", e.getMessage()); +// e.printStackTrace(); } if (StringUtil.isNullOrEmpty(stateInfo)) { LOGGER.info("register failed!"); diff --git a/src/main/java/org/bdware/datanet/router/StatusChecker.java b/src/main/java/org/bdware/datanet/router/StatusChecker.java index 226d19d2d35d40df41ee7d439bdc37388f232db5..41e2f5d37cdb1b58f4456a7d90566643b7518ac2 100644 --- a/src/main/java/org/bdware/datanet/router/StatusChecker.java +++ b/src/main/java/org/bdware/datanet/router/StatusChecker.java @@ -26,10 +26,10 @@ public class StatusChecker { private static final Logger LOGGER = LogManager.getLogger(StatusChecker.class); private static void checkLRS() { - JsonObject allLRS = Router.listValues(Global.lRSStorage); - JsonArray array = allLRS.get("data").getAsJsonArray(); - for (int i = 0; i < array.size(); i++) { - JsonObject lrs = array.get(i).getAsJsonObject(); + JsonObject allLRS = DoidDAO.listPrefix(); + JsonObject array = allLRS.get("data").getAsJsonObject(); + for (String key: array.keySet()) { + JsonObject lrs = array.get(key).getAsJsonObject(); try { if (lrs.has("address")) { String address = lrs.get("address").getAsString(); @@ -44,15 +44,19 @@ public class StatusChecker { client.connect(); if (client.isConnected()) { LOGGER.info("address" + " " + address + "---> online"); + if (!lrs.get("status").getAsString().equals("online")){ + lrs.addProperty("status", "online"); + DoidDAO.putDO(lrs.get("doId").getAsString(), lrs); + } lrs.addProperty("status", "online"); } else { - lrs.addProperty("status", "offline"); LOGGER.info("address" + " " + address + "---> offline"); + if (!lrs.get("status").getAsString().equals("offline")){ + lrs.addProperty("status", "offline"); + DoidDAO.putDO(lrs.get("doId").getAsString(), lrs); + } } client.close(); - // 如果有相同key,是覆盖掉还是? - Global.lRSStorage.put(lrs.get("doId").getAsString(), lrs.toString()); - Global.doStorage.put(lrs.get("doId").getAsString(), lrs.toString()); } } catch (Exception e) { LOGGER.error("[LRS OBJECT CHECK FAILED]: exception={}", e.getMessage(), e); @@ -63,10 +67,11 @@ public class StatusChecker { } private static void checkRepos() { - JsonObject allRepo = Router.listValues(Global.repoStorage); - JsonArray array = allRepo.get("data").getAsJsonArray(); - for (int i = 0; i < array.size(); i++) { - JsonObject repo = array.get(i).getAsJsonObject(); +// JsonObject allRepo = Router.listValues(Global.repoStorage); + JsonObject allRepo = DoidDAO.listRepo(); + JsonObject array = allRepo.get("data").getAsJsonObject(); + for (String key : array.keySet()) { + JsonObject repo = array.get(key).getAsJsonObject(); if (repo.has("address")) { try { String address = repo.get("address").getAsString(); @@ -86,14 +91,18 @@ public class StatusChecker { client.connect(config); if (client.isConnected()) { LOGGER.info("Check address" + " " + address + "---> online"); - repo.addProperty("status", "online"); + if (!repo.get("status").getAsString().equals("online")){ + repo.addProperty("status", "online"); + DoidDAO.putDO(repo.get("doId").getAsString(), repo); + } } else { LOGGER.info("Check address" + " " + address + "---> offline"); - repo.addProperty("status", "offline"); + if (!repo.get("status").getAsString().equals("offline")){ + repo.addProperty("status", "offline"); + DoidDAO.putDO(repo.get("doId").getAsString(), repo); + } } // 如果有相同key,是覆盖掉还是? - Global.repoStorage.put(repo.get("doId").getAsString(), repo.toString()); - Global.doStorage.put(repo.get("doId").getAsString(), repo.toString()); // fix: 测试发现client在未联通的状态下close会报异常,所以这里加一个判断 if (client.isConnected()) { client.close(); @@ -133,8 +142,6 @@ public class StatusChecker { e.printStackTrace(); } } - // 更改了参数:第一个参数的单位是秒,状态检查机制启动太晚不利于调试 - }, 10, 30, TimeUnit.SECONDS); - + }, 100, 30, TimeUnit.SECONDS); } } diff --git a/src/main/java/org/bdware/datanet/router/irp/CommonServerHandler.java b/src/main/java/org/bdware/datanet/router/irp/CommonServerHandler.java index 62bf1c2aa4424e9dcfde0434f29cd0960347e524..fd2d1b03d553facd1abc34b780b406cad095711e 100644 --- a/src/main/java/org/bdware/datanet/router/irp/CommonServerHandler.java +++ b/src/main/java/org/bdware/datanet/router/irp/CommonServerHandler.java @@ -9,6 +9,7 @@ import com.google.gson.JsonParser; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.datanet.router.Config; +import org.bdware.datanet.router.DoidDAO; import org.bdware.datanet.router.Global; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.irp.irplib.core.IrpMessage; @@ -52,8 +53,9 @@ public class CommonServerHandler { if (Global.repoProxy.get(repoId) != null) { repoId = Global.repoProxy.get(repoId); } - String doInfo = Global.repoStorage.get(repoId); - LOGGER.info("test:" + doInfo); +// String doInfo = Global.repoStorage.get(repoId); + String doInfo = DoidDAO.getDO(repoId); + if (doInfo != null) { JsonObject jo = JsonUtil.parseStringAsJsonObject(doInfo); if (jo.has("otherAddrs")) { @@ -105,7 +107,7 @@ public class CommonServerHandler { } public static JsonObject getTargetRepoInfo(String doid) { - String repoInfo = Global.repoStorage.get(doid); + String repoInfo = DoidDAO.getDO(doid); JsonObject info; if (repoInfo != null && repoInfo.length() > 0) { info = JsonUtil.parseStringAsJsonObject(repoInfo); @@ -131,7 +133,7 @@ public class CommonServerHandler { if (doidPrefix.equals(Global.config.getPrefix())) { return Global.config.getRootRouterInfo(); } - String repoInfo = Global.lRSStorage.get(doidPrefix); + String repoInfo = DoidDAO.getDO(doidPrefix); JsonObject info; if (repoInfo != null && repoInfo.length() > 0) { info = JsonUtil.parseStringAsJsonObject(repoInfo); @@ -240,7 +242,8 @@ public class CommonServerHandler { if (cacheDOID != null) { return CommonServerHandler.wrapResolveResponse(doid, cacheDOID); } ; - String doStr = Global.doStorage.get(doid); +// String doStr = Global.doStorage.get(doid); + String doStr = DoidDAO.getDO(doid); if (doStr == null || doStr.length() == 0) { @@ -299,7 +302,7 @@ public class CommonServerHandler { private static int appendExtraDoId(String doId, JsonObject appendixes) { if (appendixes.has(doId)) return 0; - String doStr = Global.doStorage.get(doId); + String doStr = DoidDAO.getDO(doId); if (doStr != null) { appendixes.add(doId, pruneUnusedProp(JsonParser.parseString(doStr))); return doStr.length(); @@ -307,8 +310,10 @@ public class CommonServerHandler { return 0; } + // not sure what to do with type static String[] toRemove = new String[] {"pubKey", "owner", "date", "status", "doId", "bcoId", - "repoId", "_updateTimestamp"}; + "repoId", "_updateTimestamp","type"}; + private static JsonElement pruneUnusedProp(JsonElement jsonElement) { if (jsonElement.isJsonObject()) { @@ -322,15 +327,25 @@ public class CommonServerHandler { private static JsonObject delegateResponseInternal(String doid, String targetDoid) { String parent = CommonServerHandler.getParentDoid(targetDoid); - RocksIterator iter = Global.lRSStorage.newIterator(); - for (iter.seekToFirst(); iter.isValid(); iter.next()) { - String key = new String(iter.key()); - if (parent.startsWith(key)) { - JsonObject jo = JsonUtil.parseStringAsJsonObject(new String(iter.value())); + JsonObject allLRS = DoidDAO.listPrefix(); + JsonObject array = allLRS.get("data").getAsJsonObject(); + for (String id : array.keySet()){ + if (parent.startsWith(id)) { + JsonObject jo = array.get(id).getAsJsonObject(); jo.addProperty("msgType", "delegateRequest"); return jo; } } + +// RocksIterator iter = Global.lRSStorage.newIterator(); +// for (iter.seekToFirst(); iter.isValid(); iter.next()) { +// String key = new String(iter.key()); +// if (parent.startsWith(key)) { +// JsonObject jo = JsonUtil.parseStringAsJsonObject(new String(iter.value())); +// jo.addProperty("msgType", "delegateRequest"); +// return jo; +// } +// } if (Global.config.getPrefix().contains(".")) { JsonObject jo = new JsonObject(); jo.addProperty("url", CommonServerHandler.getUpperRouterURL()); @@ -348,11 +363,12 @@ public class CommonServerHandler { String parent = CommonServerHandler.getParentDoid(targetDoid); // seek for next! if (!doid.contains("://") || doid.startsWith(Global.config.getScheme())) { - RocksIterator iter = Global.lRSStorage.newIterator(); - for (iter.seekToFirst(); iter.isValid(); iter.next()) { - String key = new String(iter.key()); - if (parent.startsWith(key)) { - JsonObject jo = JsonUtil.parseStringAsJsonObject(new String(iter.value())); +// RocksIterator iter = Global.lRSStorage.newIterator(); + JsonObject allLRS = DoidDAO.listPrefix(); + JsonObject array = allLRS.get("data").getAsJsonObject(); + for (String id : array.keySet()) { + if (parent.startsWith(id)) { + JsonObject jo = array.get(id).getAsJsonObject(); return IrpResponse.newDelegateRouterResponse(jo.get("address").getAsString()); } } diff --git a/src/main/java/org/bdware/datanet/router/irp/IrsHandlerImpl.java b/src/main/java/org/bdware/datanet/router/irp/IrsHandlerImpl.java index efb30aea93172564e20aa940b0975d8467d3d2a8..9935984a9018c2267df6641cb001169646281e68 100644 --- a/src/main/java/org/bdware/datanet/router/irp/IrsHandlerImpl.java +++ b/src/main/java/org/bdware/datanet/router/irp/IrsHandlerImpl.java @@ -8,12 +8,16 @@ import org.apache.cxf.common.util.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.datanet.router.Config; +import org.bdware.datanet.router.DoidDAO; import org.bdware.datanet.router.Global; import org.bdware.datanet.router.IDFilter; +import org.bdware.datanet.router.InfoSendTicker; +import org.bdware.datanet.router.DoidDAO; import org.bdware.doip.auditrepo.AutoAuditDO; import org.bdware.irp.irplib.core.*; import org.bdware.irp.irplib.util.EncoderUtils; import org.bdware.irp.irplib.util.IrpCommon; +import org.bdware.irp.stateinfo.StateInfoBase; import org.bdware.irp.irpserver.IrpHandler; import org.bdware.irp.irpserver.IrpServerInfo; import org.bdware.irp.stateinfo.StateInfoBase; @@ -22,10 +26,7 @@ import org.rocksdb.RocksIterator; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.UUID; +import java.util.*; // TODO we do not support such handler public class IrsHandlerImpl implements IrpHandler { @@ -58,6 +59,7 @@ public class IrsHandlerImpl implements IrpHandler { return ret; } + // todo: should we offer batch create DOID? @Override public IrpMessage batchCreateDOID(IrpMessage request) { IrpRequest req = (IrpRequest) request; @@ -75,7 +77,12 @@ public class IrsHandlerImpl implements IrpHandler { if (repoId != null) { stateInfoBase.handleValues.addProperty("owner", new String(request.credential.signerDoid, StandardCharsets.UTF_8)); - Global.doStorage.put(stateInfoBase.identifier, stateInfoBase.handleValues.toString()); + try{ + DoidDAO.putDO(stateInfoBase.identifier, stateInfoBase.handleValues); + }catch (Exception e){ + LOGGER.error("[PUT DOID FAILED]: exception={}", e.getMessage()); + } +// Global.doStorage.put(stateInfoBase.identifier, stateInfoBase.handleValues.toString()); List batch = batchGenerateDOID(req.createNumber, stateInfoBase.handleValues, repoId); @@ -94,25 +101,23 @@ public class IrsHandlerImpl implements IrpHandler { } public static void putDO(String doid, JsonObject jo) { - String timestamp = System.currentTimeMillis() + "_" + doid.hashCode(); - jo.addProperty("_updateTimestamp", timestamp); - Global.doidIndex.put(timestamp, doid); - Global.doStorage.put(doid, jo.toString()); - } - - public static void removeDO(String doid) { + // 迁移到 DoidDAO - 第一步迁移 try { - JsonObject jsonObject = - JsonParser.parseString(Global.doStorage.get(doid)).getAsJsonObject(); - Global.doStorage.delete(doid); - if (jsonObject.has("_updateTimestamp")) { - Global.doidIndex.delete(jsonObject.get("_updateTimestamp").getAsString()); - } + DoidDAO.putDO(doid, jo); } catch (Exception e) { - e.printStackTrace(); + // 如果 DoidDAO 失败,回退到原有方式(暂时保留兼容性) + LOGGER.error("[putDO error: {}]", e.getMessage()); } } + public static void updateDO(String doid, JsonObject jo){ + try { + DoidDAO.deleteDO(doid); + DoidDAO.putDO(doid, jo); + } catch (Exception e){ + LOGGER.error("[DELETE DOID FAILED]: exception={}", e.getMessage()); + } + } /* * if enableFlex = true user can register 1) prefix/repoid/uuid 2) prefix/uuid @@ -171,7 +176,7 @@ public class IrsHandlerImpl implements IrpHandler { } stateInfoBase.handleValues.addProperty("owner", new String(request.credential.signerDoid, StandardCharsets.UTF_8)); - if (isExistedDoid(stateInfoBase.identifier)) { + if (DoidDAO.isExistedDoid(stateInfoBase.identifier)) { IrpResponse resp = IrpResponse.newIrsCreateDoidResponse(stateInfoBase.identifier); resp.header.responseCode = IrpMessageCode.RC_ERROR; resp.responseMessage = EncoderUtils.encodeString( @@ -201,7 +206,7 @@ public class IrsHandlerImpl implements IrpHandler { try { if (handleValues != null && handleValues.has("repoId")) { String repoId = handleValues.get("repoId").getAsString(); - String repoInfo = Global.getRepo(repoId); + String repoInfo = DoidDAO.getDO(repoId); if (repoInfo.indexOf(owner) != -1 && repoInfo != null) return repoId; } @@ -216,12 +221,13 @@ public class IrsHandlerImpl implements IrpHandler { IrpRequest req = (IrpRequest) request; StateInfoBase stateInfoBase = request2StateInfo(req.getDoid(), req.getDoidValues()); CommonServerHandler.cacheResolve.invalidate(stateInfoBase.identifier); - if (isExistedDoid(stateInfoBase.identifier)) { + stateInfoBase.handleValues.addProperty("owner", + new String(request.credential.signerDoid, StandardCharsets.UTF_8)); + if (DoidDAO.isExistedDoid(stateInfoBase.identifier)) { // normal do if (getRepoIdFromDOHandle(stateInfoBase.handleValues, EncoderUtils.decodeString(request.credential.signerDoid)) != null) { - removeDO(stateInfoBase.identifier); - putDO(stateInfoBase.identifier, stateInfoBase.handleValues); + updateDO(stateInfoBase.identifier,stateInfoBase.handleValues); IrpResponse resp = IrpResponse.newIrsUpdateDoidResponse(stateInfoBase.identifier); resp.header.responseCode = IrpMessageCode.RC_SUCCESS; resp.responseMessage = EncoderUtils.encodeString("success"); @@ -229,8 +235,8 @@ public class IrsHandlerImpl implements IrpHandler { return resp; } else if (hasCorrectRepo(getRepoId(stateInfoBase.identifier), EncoderUtils.decodeString(request.credential.signerDoid))) { - removeDO(stateInfoBase.identifier); - putDO(stateInfoBase.identifier, stateInfoBase.handleValues); + updateDO(stateInfoBase.identifier,stateInfoBase.handleValues); + IrpResponse resp = IrpResponse.newIrsUpdateDoidResponse(stateInfoBase.identifier); resp.header.responseCode = IrpMessageCode.RC_SUCCESS; resp.responseMessage = EncoderUtils.encodeString("success"); @@ -238,8 +244,8 @@ public class IrsHandlerImpl implements IrpHandler { return resp; } else if (matchOwner(stateInfoBase.identifier, EncoderUtils.decodeString(request.credential.signerDoid))) { - removeDO(stateInfoBase.identifier); - putDO(stateInfoBase.identifier, stateInfoBase.handleValues); + updateDO(stateInfoBase.identifier, stateInfoBase.handleValues); + IrpResponse resp = IrpResponse.newIrsUpdateDoidResponse(stateInfoBase.identifier); resp.header.responseCode = IrpMessageCode.RC_SUCCESS; resp.responseMessage = EncoderUtils.encodeString("success"); @@ -263,13 +269,24 @@ public class IrsHandlerImpl implements IrpHandler { } private boolean matchOwner(String identifier, String signerDoid) { - String content = Global.doStorage.get(identifier); try { + String content = DoidDAO.getDO(identifier); if (!StringUtil.isNullOrEmpty(content)) { JsonObject jo = JsonParser.parseString(content).getAsJsonObject(); return jo.has("owner") && jo.get("owner").getAsString().equals(signerDoid); } } catch (Exception e) { + LOGGER.error("[back to previous: match Owner error]"); + try { +// String content = Global.doStorage.get(identifier); + String content = DoidDAO.getDO(identifier); + if (!StringUtil.isNullOrEmpty(content)) { + JsonObject jo = JsonParser.parseString(content).getAsJsonObject(); + return jo.has("owner") && jo.get("owner").getAsString().equals(signerDoid); + } + } catch (Exception fallbackException) { + System.err.println("Both DoidDAO and doStorage get failed in matchOwner: " + fallbackException.getMessage()); + } } return false; } @@ -284,18 +301,17 @@ public class IrsHandlerImpl implements IrpHandler { } private boolean hasCorrectRepo(String repoId, String credential) { - if (Global.repoStorage.get(repoId) != null) { - JsonObject result = - JsonParser.parseString(Global.doStorage.get(repoId)).getAsJsonObject(); - if (credential == null || !credential.equals(result.get("pubKey").getAsString())) - return false; - return true; + try{ + String content = DoidDAO.getDO(repoId); + if (!StringUtil.isNullOrEmpty(content)) { + JsonObject jo = JsonParser.parseString(content).getAsJsonObject(); + return jo.has("pubKey") && jo.get("pubKey").getAsString().equals(credential); + } + return false; + }catch (Exception e){ + LOGGER.error("[back to previous: hasCorrectRepo error]"); + return false; } - return false; - } - - private boolean isExistedDoid(String identifier) { - return Global.doStorage.get(identifier) != null; } private boolean hasPermission(IrpMessage request, JsonObject content) { @@ -324,11 +340,16 @@ public class IrsHandlerImpl implements IrpHandler { String doId = req.getDoid(); CommonServerHandler.cacheResolve.invalidate(doId); - if (isExistedDoid(doId)) { + if (DoidDAO.isExistedDoid(doId)) { JsonObject content = - JsonParser.parseString(Global.doStorage.get(req.getDoid())).getAsJsonObject(); + JsonParser.parseString(DoidDAO.getDO(doId)).getAsJsonObject(); if (hasPermission(request, content)) { - removeDO(doId); + try { + DoidDAO.deleteDO(doId); + } catch (Exception e) { + e.printStackTrace(); + LOGGER.error("[back to previous: deleteDOID error]"); + } IrpResponse resp = IrpResponse.newIrsDeleteDoidResponse(doId); resp.header.responseCode = IrpMessageCode.RC_SUCCESS; resp.result = "success"; @@ -356,32 +377,61 @@ public class IrsHandlerImpl implements IrpHandler { return stateInfoBase; } +// public IrpMessage verifyChildRouter(IrpMessage request) { +// IrpForRouterRequest req = (IrpForRouterRequest) request; +// IrpMessage res; +// LOGGER.info("req.pubkey:" + req.pubkey); +// LOGGER.info("req.name:" + req.name); +// LOGGER.info("req.signInfo:" + req.signInfo); +// LOGGER.info("verify result: " + verifySignInfo(req.pubkey, req.name, req.signInfo)); +// if (verifySignInfo(req.pubkey, req.name, req.signInfo)) { +// String prefix = DoidDAO.getNamePubkey(req.name + "|" + req.pubkey); +// if (prefix == null) { +// res = IrpResponse.newErrorResponse(IrpMessageCode.OC_VERIFY_ROUTER, +// IrpMessageCode.RC_ERROR, +// Global.config.getPrefix() + ": no such " + req.name + " -> " + req.pubkey); +// return res; +// } +// String routerInfo; +// routerInfo = DoidDAO.getDO(prefix); +// +// LOGGER.info("req routerInfo: " + routerInfo); +// +// res = IrpForRouterResponse.newVerifyRouterAuthResponse(prefix, +// getPortInAddress(routerInfo), routerInfo); +//// res = IrpResponse.newErrorResponse(IrpMessageCode.OC_VERIFY_ROUTER, +//// IrpMessageCode.RC_ERROR, +//// Global.config.getPrefix() + ": no such " + req.name + " -> " + req.pubkey); +// } else { +// res = IrpResponse.newErrorResponse(IrpMessageCode.OC_VERIFY_ROUTER, +// IrpMessageCode.RC_ERROR, +// Global.config.getPrefix() + ": verify the router signInfo failed!"); +// } +// return res; +// } @Override public IrpMessage verifyChildRouter(IrpMessage request) { IrpForRouterRequest req = (IrpForRouterRequest) request; IrpMessage res; - LOGGER.info("req.pubkey:" + req.pubkey); LOGGER.info("req.name:" + req.name); LOGGER.info("req.signInfo:" + req.signInfo); - LOGGER.info("verify result: " + verifySignInfo(req.pubkey, req.name, req.signInfo)); - if (verifySignInfo(req.pubkey, req.name, req.signInfo)) { - String prefix = Global.indexStorage.get(req.name + "|" + req.pubkey); - LOGGER.info("req prefix: " + prefix); + String doid = Arrays.toString(req.doid); + if (!DoidDAO.isExistedDoid(doid)) { + res = IrpResponse.newErrorResponse(IrpMessageCode.OC_VERIFY_ROUTER, + IrpMessageCode.RC_ERROR, + Global.config.getPrefix() + ": no such " + req.name + " -> " + req.pubkey); + return res; + } + JsonObject jo = JsonParser.parseString(DoidDAO.getDO(doid)).getAsJsonObject(); + String pubkey = jo.get("pubKey").getAsString(); + LOGGER.info("verify result: " + verifySignInfo(pubkey, Arrays.toString(req.doid), req.signInfo)); + + if (verifySignInfo(pubkey, Arrays.toString(req.doid), req.signInfo)) { String routerInfo; - if (!prefix.contains("/")) { - routerInfo = Global.lRSStorage.get(prefix); - } else - routerInfo = Global.repoStorage.get(prefix); + routerInfo = DoidDAO.getDO(doid); LOGGER.info("req routerInfo: " + routerInfo); - if (prefix != null) - res = IrpForRouterResponse.newVerifyRouterAuthResponse(prefix, - getPortInAddress(routerInfo), routerInfo); - else { - res = IrpResponse.newErrorResponse(IrpMessageCode.OC_VERIFY_ROUTER, - IrpMessageCode.RC_ERROR, - Global.config.getPrefix() + ": no such " + req.name + " -> " + req.pubkey); - } - } else { + res = IrpForRouterResponse.newVerifyRouterAuthResponse(doid,getPortInAddress(routerInfo), routerInfo); + }else{ res = IrpResponse.newErrorResponse(IrpMessageCode.OC_VERIFY_ROUTER, IrpMessageCode.RC_ERROR, Global.config.getPrefix() + ": verify the router signInfo failed!"); @@ -400,7 +450,7 @@ public class IrsHandlerImpl implements IrpHandler { JsonObject arg = new JsonObject(); arg.add("routerInfo", Global.config.getRootRouterInfo()); String auditProxy = AutoAuditDO.getAuditProxyDoid(Global.config.getPrefix()); - String auditProxyInfo = Global.repoStorage.get(auditProxy); + String auditProxyInfo = DoidDAO.getDO(auditProxy); if (auditProxyInfo != null) arg.addProperty(AutoAuditDO.AUDIT_PROXY, auditProxyInfo); res.doidValues = IrpMessage.fromJson2Byte(arg); @@ -425,24 +475,24 @@ public class IrsHandlerImpl implements IrpHandler { return IrpResponse.newErrorResponse(IrpMessageCode.OC_QUERY_ROUTER, IrpMessageCode.RC_ERROR, "count <" + IrpCommon.MAX_ARRAY_LENGTH_SIZE); } + Long startTs = value.has("startTs")?value.get("startTs").getAsLong():null; + Long endTs = value.has("endTs")?value.get("endTs").getAsLong():null; JsonObject arg; boolean ascending = true; if (value.has("order")) { ascending = !value.get("order").getAsString().equals("desc"); } if (value.has("queryByCreateTime") && value.get("queryByCreateTime").getAsBoolean()) { - arg = queryIdentifierByOffsetByTimeIndex(offset, count, IDFilter.defaultFilter(), - ascending); + arg = queryIdentifierByOffsetByTimeIndex(startTs, endTs,offset, count, ascending); } else { - arg = queryIdentifierByOffsetByRocksDB(offset, count, IDFilter.defaultFilter(), - ascending); +// arg = queryIdentifierByOffsetByRocksDB(startTs, endTs, offset, count, ascending); + arg = queryIdentifierByOffsetByTimeIndex(startTs, endTs, offset, count, ascending); } IrpResponse res = new IrpResponse(IrpMessageCode.OC_QUERY_ROUTER, IrpMessageCode.RC_SUCCESS); res.requestDigestNeeded = false; res.doidValues = IrpMessage.fromJson2Byte(arg); return res; - } catch (Exception e) { e.printStackTrace(); return IrpResponse.newErrorResponse(IrpMessageCode.OC_QUERY_ROUTER, @@ -450,107 +500,67 @@ public class IrsHandlerImpl implements IrpHandler { } } - public static JsonObject queryIdentifierByOffsetByTimeIndex(int offset, int count, - IDFilter filter, boolean ascending) { - RocksIterator iter = null; - try { - iter = Global.doidIndex.newIterator(); - if (ascending) - iter.seekToFirst(); - else - iter.seekToLast(); - if (!iter.isValid()) { - tryReIndex(); - iter = Global.doidIndex.newIterator(); - iter.seekToFirst(); - } - int total = 0; - JsonArray array = new JsonArray(); - JsonObject arg = new JsonObject(); - int ccount = 0; - for (; iter.isValid();) { - String id = new String(iter.value(), StandardCharsets.UTF_8); - if (total >= offset && total < offset + count) { - if (filter.accept(id)) { - ccount++; - array.add(id); - } - } - if (filter.accept(id)) - total++; - if (ascending) - iter.next(); - else - iter.prev(); - } - arg.addProperty("total", total); - arg.addProperty("count", ccount); - arg.addProperty("status", "success"); - arg.add("data", array); - return arg; - } finally { - if (iter != null) - iter.close(); - } + public static JsonObject queryIdentifierByOffsetByTimeIndex(Long startTs,Long endTs, int offset, int count, boolean ascending) throws Exception { + JsonArray array; + JsonObject arg = new JsonObject(); + array = DoidDAO.queryDataDoidList(startTs,endTs,offset, count, ascending); + arg.addProperty("total", DoidDAO.getCount("data")); + arg.addProperty("count", array.size()); + arg.addProperty("status", "success"); + arg.add("data", array); + return arg; } +// private static void tryReIndex() { +// try { +// LOGGER.info("tryReIndex, estimate key size:" + Global.doStorage.estimateKeySize()); +// RocksIterator iter = Global.doStorage.newIterator(); +// iter.seekToFirst(); +// for (; iter.isValid();) { +// String doid = new String(iter.key(), StandardCharsets.UTF_8); +// JsonObject jo = +// JsonParser.parseString(new String(iter.value(), StandardCharsets.UTF_8)) +// .getAsJsonObject(); +// putDO(doid, jo); +// iter.next(); +// } +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } + + public static JsonObject queryDataByOffsetByRocksDB(Long startTs,Long endTs,int offset, int count, boolean ascending) throws Exception { + JsonObject ret = new JsonObject(); + JsonObject jo; + jo = DoidDAO.queryDataList(startTs,endTs,offset, count, ascending); + ret.addProperty("status", "success"); + ret.addProperty("total", DoidDAO.getCount("data")); + ret.addProperty("count", jo.size()); + ret.add("data", jo); + return ret; + } - private static void tryReIndex() { - try { - LOGGER.info("tryReIndex, estimate key size:" + Global.doStorage.estimateKeySize()); - RocksIterator iter = Global.doStorage.newIterator(); - iter.seekToFirst(); - for (; iter.isValid();) { - String doid = new String(iter.key(), StandardCharsets.UTF_8); - JsonObject jo = - JsonParser.parseString(new String(iter.value(), StandardCharsets.UTF_8)) - .getAsJsonObject(); - IrsHandlerImpl.putDO(doid, jo); - iter.next(); - } - } catch (Exception e) { - e.printStackTrace(); - } + public static JsonObject queryLRSByOffsetByRocksDB(Long startTs,Long endTs,int offset, int count, boolean ascending) throws Exception{ + JsonObject ja; + ja = DoidDAO.queryPrefixList(startTs, endTs, offset, count, ascending); + return ja; + } + public static JsonObject queryLRSByDoid(String doid ,int offset, int count, boolean ascending) throws Exception{ + JsonObject ja; + ja = DoidDAO.queryPrefixListInSearchMode(doid, offset, count, ascending); + return ja; } - public static JsonObject queryIdentifierByOffsetByRocksDB(int offset, int count, - IDFilter filter, boolean ascending) { - RocksIterator iter = null; - try { - iter = Global.doStorage.newIterator(); - if (ascending) - iter.seekToFirst(); - else - iter.seekToLast(); - int total = 0; - JsonArray array = new JsonArray(); - JsonObject arg = new JsonObject(); - int ccount = 0; - for (; iter.isValid();) { - String id = new String(iter.key(), StandardCharsets.UTF_8); - if (total >= offset && total < offset + count) { - if (filter.accept(id)) { - ccount++; - array.add(id); - } + public static JsonObject queryRepositoryByOffsetByRocksDB(String type,Long startTs,Long endTs,int offset,int count ,boolean ascending)throws Exception { + JsonObject ja; + ja = DoidDAO.queryRepoList(type, startTs, endTs, offset, count, ascending); + return ja; + } - } - if (filter.accept(id)) - total++; - if (ascending) - iter.next(); - else - iter.prev(); - } - arg.addProperty("total", total); - arg.addProperty("count", ccount); - arg.addProperty("status", "success"); - arg.add("data", array); - return arg; - } finally { - if (iter != null) - iter.close(); - } + public static JsonObject queryRepositoryByDoid(String doid,int offset,int count ,boolean ascending)throws Exception { + JsonObject ja; + ja = DoidDAO.queryRepoListInSearchMode(doid, offset, count, ascending); + return ja; } private int getPortInAddress(String routerInfo) { @@ -565,15 +575,14 @@ public class IrsHandlerImpl implements IrpHandler { } } - public boolean verifySignInfo(String pubkey, String name, String signInfo) { +// public boolean verifySignInfo(String pubkey, String name, String signInfo) { +// // todo:need the type parameter +// return Config.verify(pubkey, name + "|" + pubkey, signInfo); +// } + + public boolean verifySignInfo(String pubkey, String doid, String signInfo) { // todo:need the type parameter - // if (Global.lRSStorage.get(pubkey) == null && Global.repoStorage.get(pubkey) == null) { - // //the child router/repo is not registry in the router - // return false; - // } else { - // return Config.verify(pubkey, name + "|" + pubkey, signInfo); - // } - return Config.verify(pubkey, name + "|" + pubkey, signInfo); + return Config.verify(pubkey, doid, signInfo); } @Override diff --git a/src/main/java/org/bdware/datanet/router/irp/JsonArraySorter.java b/src/main/java/org/bdware/datanet/router/irp/JsonArraySorter.java new file mode 100644 index 0000000000000000000000000000000000000000..64dbec0f10beebbdc381b48bd427aa7b53b5cc5b --- /dev/null +++ b/src/main/java/org/bdware/datanet/router/irp/JsonArraySorter.java @@ -0,0 +1,65 @@ +package org.bdware.datanet.router.irp; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import java.util.ArrayList; +import java.util.List; + +public class JsonArraySorter { + /** + * 对任意 JsonArray 按指定字段升/降序排序。 + * + * @param array 待排序的 JsonArray(内部元素必须是 JsonObject) + * @param sortBy 排序字段名 + * @param ascending true=升序, false=降序 + * @return 一个新的、已排序的 JsonArray + */ + public static JsonArray sort(JsonArray array, + String sortBy, + boolean ascending) { + List list = new ArrayList<>(); + for (JsonElement e : array) { + list.add(e.getAsJsonObject()); + } + + // 排序逻辑 + list.sort((o1, o2) -> { + JsonElement v1 = o1.get(sortBy); + JsonElement v2 = o2.get(sortBy); + + // Null 或者 字段缺失时视为相等 + if (v1 == null || v2 == null) { + return 0; + } + + int cmp; + // 数字 vs 数字 + if (v1.isJsonPrimitive() && v2.isJsonPrimitive()) { + JsonPrimitive p1 = v1.getAsJsonPrimitive(); + JsonPrimitive p2 = v2.getAsJsonPrimitive(); + + if (p1.isNumber() && p2.isNumber()) { + cmp = Long.compare(p1.getAsLong(), p2.getAsLong()); + } else { + // 字符串比较 + cmp = p1.getAsString().compareTo(p2.getAsString()); + } + } else { + // 其他类型转成字符串比 + cmp = v1.toString().compareTo(v2.toString()); + } + + return ascending ? cmp : -cmp; + }); + + // 构造新的 JsonArray 返回 + JsonArray sorted = new JsonArray(); + for (JsonObject obj : list) { + sorted.add(obj); + } + return sorted; + } +} diff --git a/src/main/java/org/bdware/datanet/router/util/JsonUtils.java b/src/main/java/org/bdware/datanet/router/util/JsonUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..f69a7dcdfdf20daa570fe09b56d4fb58de3d50d4 --- /dev/null +++ b/src/main/java/org/bdware/datanet/router/util/JsonUtils.java @@ -0,0 +1,110 @@ +package org.bdware.datanet.router.util; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +/** + * JSON解析工具类,提供安全的null值检查 + */ +public class JsonUtils { + + /** + * 安全获取整数值,带默认值 + * @param jo JSON对象 + * @param key 键名 + * @param defaultValue 默认值 + * @return 整数值或默认值 + */ + public static int getIntSafely(JsonObject jo, String key, int defaultValue) { + if (!jo.has(key)) { + return defaultValue; + } + JsonElement element = jo.get(key); + if (element == null || element.isJsonNull()) { + return defaultValue; + } + try { + return element.getAsInt(); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * 安全获取长整数值,带默认值 + * @param jo JSON对象 + * @param key 键名 + * @param defaultValue 默认值 + * @return 长整数值或默认值 + */ + public static long getLongSafely(JsonObject jo, String key, long defaultValue) { + if (!jo.has(key)) { + return defaultValue; + } + JsonElement element = jo.get(key); + if (element == null || element.isJsonNull()) { + return defaultValue; + } + try { + return element.getAsLong(); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * 安全获取字符串值,带默认值 + * @param jo JSON对象 + * @param key 键名 + * @param defaultValue 默认值 + * @return 字符串值或默认值 + */ + public static String getStringSafely(JsonObject jo, String key, String defaultValue) { + if (!jo.has(key)) { + return defaultValue; + } + JsonElement element = jo.get(key); + if (element == null || element.isJsonNull()) { + return defaultValue; + } + try { + return element.getAsString(); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * 安全获取布尔值,带默认值 + * @param jo JSON对象 + * @param key 键名 + * @param defaultValue 默认值 + * @return 布尔值或默认值 + */ + public static boolean getBooleanSafely(JsonObject jo, String key, boolean defaultValue) { + if (!jo.has(key)) { + return defaultValue; + } + JsonElement element = jo.get(key); + if (element == null || element.isJsonNull()) { + return defaultValue; + } + try { + return element.getAsBoolean(); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * 检查排序方向,返回是否为升序 + * @param jo JSON对象 + * @param key 键名(通常是"order") + * @param defaultAscending 默认是否升序 + * @return true表示升序,false表示降序 + */ + public static boolean isAscendingOrder(JsonObject jo, String key, boolean defaultAscending) { + String order = getStringSafely(jo, key, defaultAscending ? "asc" : "desc"); + return !"desc".equalsIgnoreCase(order); + } +} \ No newline at end of file diff --git "a/src/main/resources/imgs/\346\235\203\351\231\220-\344\270\273\344\275\223\345\210\206\346\236\220.png" "b/src/main/resources/imgs/\346\235\203\351\231\220-\344\270\273\344\275\223\345\210\206\346\236\220.png" new file mode 100644 index 0000000000000000000000000000000000000000..ea6fd801cd97fddf84721cb74621c85ad367e127 Binary files /dev/null and "b/src/main/resources/imgs/\346\235\203\351\231\220-\344\270\273\344\275\223\345\210\206\346\236\220.png" differ diff --git "a/src/main/resources/imgs/\346\235\203\351\231\220-\346\224\273\345\207\273.png" "b/src/main/resources/imgs/\346\235\203\351\231\220-\346\224\273\345\207\273.png" new file mode 100644 index 0000000000000000000000000000000000000000..b722fd92c92ec3c717a632ab070b2bbafb987bfd Binary files /dev/null and "b/src/main/resources/imgs/\346\235\203\351\231\220-\346\224\273\345\207\273.png" differ diff --git "a/src/main/resources/imgs/\346\235\203\351\231\220-\351\227\256\351\242\230.png" "b/src/main/resources/imgs/\346\235\203\351\231\220-\351\227\256\351\242\230.png" new file mode 100644 index 0000000000000000000000000000000000000000..7ec3d085ede4fed251fa51eeaf09cac4b08d2858 Binary files /dev/null and "b/src/main/resources/imgs/\346\235\203\351\231\220-\351\227\256\351\242\230.png" differ diff --git a/src/test/java/org/bdware/router/deprecated/StorageTest.java b/src/test/java/org/bdware/router/deprecated/StorageTest.java index 33958a6b0e44806fc70066217a9bb418ab7075f6..3646528e4907a656eb6ab163ebaf035f157d876e 100644 --- a/src/test/java/org/bdware/router/deprecated/StorageTest.java +++ b/src/test/java/org/bdware/router/deprecated/StorageTest.java @@ -1,6 +1,9 @@ package org.bdware.router.deprecated; +import com.google.gson.JsonObject; +import org.bdware.datanet.router.DoidDAO; import org.bdware.datanet.router.Global; +import org.bdware.sc.util.JsonUtil; import org.junit.Test; import org.rocksdb.RocksIterator; @@ -9,19 +12,25 @@ public class StorageTest { @Test public void createPrefix() { String routerInfo = - "{\"date\": \"2021-12-17\",\"name\": \"router1\",\"doId\": \"86.139.123\",\"address\": \"106.120.124.103\",\"status\": \"online\",\"protocol\": \"IRP\",\"pubKey\": \"04da01345770b7e09d4774bf6c0395399b18814aa0b7b158f64b634b8f3d628d9964af6523835225af11e467271f4969d67bf90b32eaa09f517c79b2d1f9b8a926\"}"; + "{\"date\": \"2021-12-17\",\"name\": \"router1\",\"doId\": \"86.139.123\",\"address\": \"106.120.124.103\",\"type\": \"prefix\",\"status\": \"online\",\"protocol\": \"IRP\",\"pubKey\": \"04da01345770b7e09d4774bf6c0395399b18814aa0b7b158f64b634b8f3d628d9964af6523835225af11e467271f4969d67bf90b32eaa09f517c79b2d1f9b8a926\"}"; String doId = "86.139.123"; - Global.lRSStorage.put(doId, routerInfo); + JsonObject jo = JsonUtil.parseStringAsJsonObject(routerInfo); + try{ + DoidDAO.putDO(doId, jo); + } catch (Exception e){ + e.printStackTrace(); + } } @Test public void clearPrefix() { - RocksIterator iter = Global.lRSStorage.newIterator(); - for (iter.seekToFirst(); iter.isValid(); iter.next()) { - String jsonStr = new String(iter.key()); - System.out.println("deleteKey: " + jsonStr); - String result = Global.lRSStorage.delete(jsonStr); - System.out.println("result: " + result); - } +// RocksIterator iter = Global.lRSStorage.newIterator(); +// for (iter.seekToFirst(); iter.isValid(); iter.next()) { +// String jsonStr = new String(iter.key()); +// System.out.println("deleteKey: " + jsonStr); +// String result = Global.lRSStorage.delete(jsonStr); +// System.out.println("result: " + result); +// } + DoidDAO.clearAllStorage(); } } diff --git a/src/test/java/org/bdware/router/ui/ColumnFamily.java b/src/test/java/org/bdware/router/ui/ColumnFamily.java new file mode 100644 index 0000000000000000000000000000000000000000..9f67ba8e49d3f4f7d99de9faeb72a117e6705dfe --- /dev/null +++ b/src/test/java/org/bdware/router/ui/ColumnFamily.java @@ -0,0 +1,299 @@ +package org.bdware.router.ui; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.bdware.datanet.router.DoidDAO; +import org.rocksdb.*; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * DoidDAO 测试类 + * 测试 DoidDAO 的各种功能,包括增删改查、索引查询等 + */ +public class ColumnFamily { + + static { RocksDB.loadLibrary();} + + public static void main(String[] args) throws Exception{ + System.out.println("=== DoidDAO 测试开始 ==="); + + // 初始化 DoidDAO + DoidDAO dao = new DoidDAO(); + System.out.println("✓ DoidDAO 初始化成功"); + + // 测试基本的增删改查 + testBasicOperations(); + + // 测试不同类型的数据 + testDifferentTypes(); + + // 测试查询功能 + testQueryOperations(); + + // 测试特殊类型(system 和 code) + testSpecialTypes(); + + // 测试边界情况 + testEdgeCases(); + + System.out.println("=== 所有测试完成 ==="); + } + + /** + * 测试基本的增删改查操作 + */ + public static void testBasicOperations() throws Exception { + System.out.println("\n--- 测试基本操作 ---"); + + // 1. 测试插入数据 + String doid1 = "data/test/user001"; + JsonObject user1 = new JsonObject(); + user1.addProperty("name", "张三"); + user1.addProperty("age", 25); + user1.addProperty("email", "zhangsan@example.com"); + user1.addProperty("type", "user"); + + DoidDAO.putDO(doid1, user1); + System.out.println("✓ 插入用户数据: " + doid1); + + // 2. 测试获取数据 + byte[] data = DoidDAO.get(doid1); + if (data != null) { + String jsonStr = new String(data, StandardCharsets.UTF_8); + System.out.println("✓ 获取数据成功: " + jsonStr); + } else { + System.out.println("✗ 获取数据失败"); + } + + // 3. 测试插入更多数据 + String doid2 = "data/test/user002"; + JsonObject user2 = new JsonObject(); + user2.addProperty("name", "李四"); + user2.addProperty("age", 30); + user2.addProperty("email", "lisi@example.com"); + user2.addProperty("type", "user"); + + DoidDAO.putDO(doid2, user2); + System.out.println("✓ 插入用户数据: " + doid2); + + // 等待一秒,确保时间戳不同 + Thread.sleep(1000); + + String doid3 = "data/test/product001"; + JsonObject product1 = new JsonObject(); + product1.addProperty("name", "笔记本电脑"); + product1.addProperty("price", 5999.99); + product1.addProperty("category", "电子产品"); + product1.addProperty("type", "product"); + + DoidDAO.putDO(doid3, product1); + System.out.println("✓ 插入产品数据: " + doid3); + } + + /** + * 测试不同类型的数据 + */ + public static void testDifferentTypes() throws Exception { + System.out.println("\n--- 测试不同类型数据 ---"); + + // 测试默认类型(不指定type) + String doid = "data/test/default001"; + JsonObject defaultObj = new JsonObject(); + defaultObj.addProperty("content", "这是默认类型的数据"); + + DoidDAO.putDO(doid, defaultObj); + System.out.println("✓ 插入默认类型数据: " + doid); + + // 验证默认类型被设置为 "data" + byte[] data = DoidDAO.get(doid); + if (data != null) { + String jsonStr = new String(data, StandardCharsets.UTF_8); + System.out.println("✓ 默认类型数据: " + jsonStr); + } + + // 测试文档类型 + String docDoid = "data/test/document001"; + JsonObject document = new JsonObject(); + document.addProperty("title", "技术文档"); + document.addProperty("content", "这是一份重要的技术文档"); + document.addProperty("type", "document"); + + DoidDAO.putDO(docDoid, document); + System.out.println("✓ 插入文档数据: " + docDoid); + } + + /** + * 测试查询操作 + */ + public static void testQueryOperations() throws Exception { + System.out.println("\n--- 测试查询操作 ---"); + + // 1. 测试按类型查询 + System.out.println("查询所有 user 类型数据:"); + JsonArray users = DoidDAO.query("user", null, null, 0, 10, false); + if (users != null) { + System.out.println("✓ 找到 " + users.size() + " 个用户"); + for (int i = 0; i < users.size(); i++) { + System.out.println(" - " + users.get(i).toString()); + } + } + + // 2. 测试按类型查询(升序) + System.out.println("\n查询所有 user 类型数据(升序):"); + JsonArray usersAsc = DoidDAO.query("user", null, null, 0, 10, true); + if (usersAsc != null) { + System.out.println("✓ 找到 " + usersAsc.size() + " 个用户(升序)"); + } + + // 3. 测试查询所有数据类型 + System.out.println("\n查询所有 data 类型数据:"); + JsonArray dataItems = DoidDAO.query("data", null, null, 0, 10, false); + if (dataItems != null) { + System.out.println("✓ 找到 " + dataItems.size() + " 个数据项"); + } + + // 4. 测试分页查询 + System.out.println("\n测试分页查询(offset=1, count=1):"); + JsonArray pagedUsers = DoidDAO.query("user", null, null, 1, 1, false); + if (pagedUsers != null) { + System.out.println("✓ 分页查询结果: " + pagedUsers.size() + " 个项目"); + } + + // 5. 测试时间范围查询 + long currentTime = System.currentTimeMillis(); + long oneHourAgo = currentTime - 3600000; // 1小时前 + + System.out.println("\n测试时间范围查询(最近1小时):"); + JsonArray recentData = DoidDAO.query("user", oneHourAgo, currentTime, 0, 10, false); + if (recentData != null) { + System.out.println("✓ 最近1小时的数据: " + recentData.size() + " 个项目"); + } + } + + /** + * 测试特殊类型(system 和 code) + */ + public static void testSpecialTypes() throws Exception { + System.out.println("\n--- 测试特殊类型 ---"); + + // 测试 system 类型 + String systemDoid = "system/config/database"; + JsonObject systemConfig = new JsonObject(); + systemConfig.addProperty("host", "localhost"); + systemConfig.addProperty("port", 3306); + systemConfig.addProperty("type", "system/config"); + + DoidDAO.putDO(systemDoid, systemConfig); + System.out.println("✓ 插入系统配置: " + systemDoid); + + // 测试 code 类型 + String codeDoid = "code/java/utils"; + JsonObject codeObj = new JsonObject(); + codeObj.addProperty("language", "java"); + codeObj.addProperty("function", "utility"); + codeObj.addProperty("type", "code/java"); + + DoidDAO.putDO(codeDoid, codeObj); + System.out.println("✓ 插入代码数据: " + codeDoid); + + // 查询 system 类型数据 + System.out.println("\n查询 system 类型数据:"); + JsonArray systemData = DoidDAO.queryRepoDoidByByTimeIndex("", null, null, 0, 10, false); + if (systemData != null) { + System.out.println("✓ 找到 " + systemData.size() + " 个系统数据"); + for (int i = 0; i < systemData.size(); i++) { + System.out.println(" - " + systemData.get(i).toString()); + } + } + + // 查询 code 类型数据 + System.out.println("\n查询 code 类型数据:"); + JsonArray codeData = DoidDAO.queryCodeDoidByTimeIndex("", null, null, 0, 10, false); + if (codeData != null) { + System.out.println("✓ 找到 " + codeData.size() + " 个代码数据"); + for (int i = 0; i < codeData.size(); i++) { + System.out.println(" - " + codeData.get(i).toString()); + } + } + } + + /** + * 测试边界情况和错误处理 + */ + public static void testEdgeCases() throws Exception { + System.out.println("\n--- 测试边界情况 ---"); + + // 1. 测试获取不存在的数据 + byte[] notFound = DoidDAO.get("nonexistent/doid"); + if (notFound == null) { + System.out.println("✓ 正确处理不存在的数据"); + } else { + System.out.println("✗ 不存在的数据返回了结果"); + } + + // 2. 测试空类型查询 + JsonArray emptyTypeQuery = DoidDAO.query("", null, null, 0, 10, false); + if (emptyTypeQuery != null) { + System.out.println("✓ 空类型查询返回 " + emptyTypeQuery.size() + " 个结果"); + } + + // 3. 测试查询不存在的类型 + JsonArray nonexistentType = DoidDAO.query("nonexistent", null, null, 0, 10, false); + if (nonexistentType != null) { + System.out.println("✓ 不存在类型查询返回 " + nonexistentType.size() + " 个结果"); + } + + // 4. 测试删除操作(注意:当前删除实现有问题,这里只是演示) + System.out.println("\n测试删除操作:"); + String testDeleteDoid = "data/test/delete_me"; + JsonObject deleteObj = new JsonObject(); + deleteObj.addProperty("content", "这个数据将被删除"); + deleteObj.addProperty("type", "test"); + + DoidDAO.putDO(testDeleteDoid, deleteObj); + System.out.println("✓ 插入待删除数据: " + testDeleteDoid); + + try { + DoidDAO.deleteDO(testDeleteDoid); + System.out.println("✓ 删除操作执行完成"); + + // 验证删除 + byte[] deletedData = DoidDAO.get(testDeleteDoid); + if (deletedData == null) { + System.out.println("✓ 数据已成功删除"); + } else { + System.out.println("⚠ 数据仍然存在(可能是索引删除问题)"); + } + } catch (Exception e) { + System.out.println("⚠ 删除操作异常: " + e.getMessage()); + } + + // 5. 测试删除不存在的数据 + try { + DoidDAO.deleteDO("nonexistent/doid"); + System.out.println("✗ 删除不存在数据应该抛出异常"); + } catch (Exception e) { + System.out.println("✓ 正确抛出异常: " + e.getMessage()); + } + } +} + + + + + + + + + + + + + + + + diff --git a/src/test/java/org/bdware/router/ui/ColumnFamilyEffiTest.java b/src/test/java/org/bdware/router/ui/ColumnFamilyEffiTest.java new file mode 100644 index 0000000000000000000000000000000000000000..8590ba509f1c00c74970ddd26e078b5f94e10cd0 --- /dev/null +++ b/src/test/java/org/bdware/router/ui/ColumnFamilyEffiTest.java @@ -0,0 +1,196 @@ +package org.bdware.router.ui; + +import org.rocksdb.*; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ColumnFamilyEffiTest { + static { RocksDB.loadLibrary(); } + + private static final int N_DATA = 500_000; + private static final int N_PREF = 100; + private static final int N_SYS = 100; + // 前缀长度(包括“prefix_”或“system_”的所有字符) + private static final int PREFIX_LEN = "prefix_".getBytes().length; + + public static void main(String[] args) throws Exception { + testSingleCF("./db_singlecf"); + testMultiCF ("./db_multicf"); + } + + // ------------------------------------------------- + // 单表方案:一个 CF + BloomFilter + FixedPrefix + 范围上下界 + // ------------------------------------------------- + private static void testSingleCF(String path) throws Exception { + destroy(path); + + // 1) 启用 Statistics + Statistics stats = new Statistics(); + DBOptions dbOpts = new DBOptions() + .setCreateIfMissing(true) + .setStatistics(stats); + + // 2) 配置表格式:Bloom + whole-key filtering + BlockBasedTableConfig tblCfg = new BlockBasedTableConfig() + .setFilterPolicy(new BloomFilter(10, false)) + .setWholeKeyFiltering(true); + + // 3) 在 ColumnFamilyOptions 里加上 FixedPrefixTransform + ColumnFamilyOptions cfOpts = new ColumnFamilyOptions() + .setTableFormatConfig(tblCfg); + + List cfDesc = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts) + ); + List cfHandles = new java.util.ArrayList<>(); + + try (RocksDB db = RocksDB.open(dbOpts, path, cfDesc, cfHandles)) { + ColumnFamilyHandle cf = cfHandles.get(0); + writeTestData(db, cf); + db.flush(new FlushOptions().setWaitForFlush(true)); + + System.out.println("\n===== [SingleCF] ====="); + + // -- 单条查询 prefix_* + String singleKey = String.format("prefix_%013d_0", + System.currentTimeMillis() % 1000); + stats.reset(); + long t0 = System.nanoTime(); + db.get(cf, singleKey.getBytes()); + long t1 = System.nanoTime(); + printReport("单条查询", t1 - t0, stats); + + // -- 范围查询 prefix_* + stats.reset(); + // 构造上下界 + byte[] lower = "prefix_".getBytes(StandardCharsets.UTF_8); + byte[] upper = "prefix_\uFFFF".getBytes(StandardCharsets.UTF_8); + ReadOptions ro = new ReadOptions() + .setPrefixSameAsStart(true) + .setIterateLowerBound(new Slice(lower)) + .setIterateUpperBound(new Slice(upper)); + + long t2 = System.nanoTime(); + try (RocksIterator it = db.newIterator(cf, ro)) { + it.seek(lower); + while (it.isValid()) { + it.next(); + } + } + long t3 = System.nanoTime(); + printReport("范围查询", t3 - t2, stats); + } + } + + private static void testMultiCF(String path) throws Exception { + destroy(path); + + // 初始化 DB 与统计 + Statistics stats = new Statistics(); + DBOptions dbOpts = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true) + .setStatistics(stats); + + List cfDesc = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, new ColumnFamilyOptions()), + new ColumnFamilyDescriptor("CF_data".getBytes(), new ColumnFamilyOptions()), + new ColumnFamilyDescriptor("CF_prefix".getBytes(), new ColumnFamilyOptions()), + new ColumnFamilyDescriptor("CF_system".getBytes(), new ColumnFamilyOptions()) + ); + List cfHandles = new ArrayList<>(); + + try (RocksDB db = RocksDB.open(dbOpts, path, cfDesc, cfHandles)) { + ColumnFamilyHandle cfData = cfHandles.get(1); + ColumnFamilyHandle cfPrefix = cfHandles.get(2); + ColumnFamilyHandle cfSystem = cfHandles.get(3); + + // 写入并 flush + writeByTable(db, cfData, cfPrefix, cfSystem); + db.flush(new FlushOptions().setWaitForFlush(true)); + + System.out.println("\n===== [MultiCF] ====="); + + // 1) 从 CF_prefix 里取到一个实际存在的 prefix Key + byte[] sampleKey; + try (RocksIterator it = db.newIterator(cfPrefix)) { + it.seekToFirst(); + if (!it.isValid()) { + throw new IllegalStateException("CF_prefix 没有任何数据"); + } + sampleKey = it.key(); + } + + // 2) 单条查询 + stats.reset(); + long t0 = System.nanoTime(); + db.get(cfPrefix, sampleKey); + long t1 = System.nanoTime(); + printReport("单条查询", t1 - t0, stats); + + // 3) 范围查询:只扫 “prefix_” 段 + stats.reset(); + byte[] lower = "prefix_".getBytes(StandardCharsets.UTF_8); + byte[] upper = "prefix_\uFFFF".getBytes(StandardCharsets.UTF_8); + ReadOptions ro = new ReadOptions() + .setIterateLowerBound(new Slice(lower)) + .setIterateUpperBound(new Slice(upper)); + + long t2 = System.nanoTime(); + int i = 0; + try (RocksIterator it = db.newIterator(cfPrefix, ro)) { + it.seek(lower); + while (it.isValid()) { + i ++; + it.next(); + } + } + System.out.println("[MultiCF] 扫描了 " + i + " 条数据"); + long t3 = System.nanoTime(); + printReport("范围查询", t3 - t2, stats); + } + } + + + private static void writeTestData(RocksDB db, ColumnFamilyHandle cf) throws RocksDBException { + try (WriteBatch batch = new WriteBatch(); WriteOptions w = new WriteOptions()) { + long base = System.currentTimeMillis(); + for (int i = 0; i < N_DATA; i++) + batch.put(cf, ("data_" + (base + i % 1000) + "_" + i).getBytes(), new byte[]{1}); + for (int i = 0; i < N_PREF; i++) + batch.put(cf, ("prefix_" + (base + i % 1000) + "_" + i).getBytes(), new byte[]{2}); + for (int i = 0; i < N_SYS; i++) + batch.put(cf, ("system_" + (base + i % 1000) + "_" + i).getBytes(), new byte[]{3}); + db.write(w, batch); + } + } + + private static void writeByTable(RocksDB db, ColumnFamilyHandle d, ColumnFamilyHandle p, ColumnFamilyHandle s) + throws RocksDBException { + try (WriteBatch batch = new WriteBatch(); WriteOptions w = new WriteOptions()) { + long base = System.currentTimeMillis(); + for (int i = 0; i < N_DATA; i++) + batch.put(d, ("data_" + (base + i % 1000) + "_" + i).getBytes(), new byte[]{1}); + for (int i = 0; i < N_PREF; i++) + batch.put(p, ("prefix_" + (base + i % 1000) + "_" + i).getBytes(), new byte[]{2}); + for (int i = 0; i < N_SYS; i++) + batch.put(s, ("system_" + (base + i % 1000) + "_" + i).getBytes(), new byte[]{3}); + db.write(w, batch); + } + } + + private static void printReport(String tag, long nano, Statistics stats) { + long fileOpens = stats.getTickerCount(TickerType.NO_FILE_OPENS); + long hits = stats.getTickerCount(TickerType.BLOCK_CACHE_HIT); + long misses = stats.getTickerCount(TickerType.BLOCK_CACHE_MISS); + System.out.printf("%s: elapsed=%.3fms, fileOpens=%d, cacheHit=%d, miss=%d%n", + tag, nano / 1e6, fileOpens, hits, misses); + } + + private static void destroy(String path) throws RocksDBException { + RocksDB.destroyDB(path, new Options()); + } +} diff --git a/src/test/java/org/bdware/router/ui/DoidDAOComprehensiveTest.java b/src/test/java/org/bdware/router/ui/DoidDAOComprehensiveTest.java new file mode 100644 index 0000000000000000000000000000000000000000..2b3fa7de1a09324bdbae1011e3cddd4f7d0becc2 --- /dev/null +++ b/src/test/java/org/bdware/router/ui/DoidDAOComprehensiveTest.java @@ -0,0 +1,745 @@ +package org.bdware.router.ui; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.bdware.datanet.router.DoidDAO; +import org.junit.*; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.*; + +import static org.junit.Assert.*; + +/** + * DoidDAO 全面测试类 - 覆盖所有新功能 + * 包括多 ColumnFamily 设计、索引系统、计数功能、批量操作等 + */ +public class DoidDAOComprehensiveTest { + + private static DoidDAO dao; + private static final String TEST_DB_PATH = "./testDoStorage"; + + // 性能测试数据量配置 + private static final int SMALL_BATCH_SIZE = 50; + private static final int MEDIUM_BATCH_SIZE = 200; + private static final int LARGE_BATCH_SIZE = 500; + + @BeforeClass + public static void setUp() throws Exception { + // 清理测试数据库 + cleanupTestDB(); + + // 创建 DAO 实例 + dao = new DoidDAO(); + + System.out.println("=== DoidDAO 全面测试开始 ==="); + System.out.println("测试数据库路径: " + TEST_DB_PATH); + } + + @AfterClass + public static void tearDown() throws Exception { + if (dao != null) { + DoidDAO.close(); + } + // 清理测试数据库 + cleanupTestDB(); + System.out.println("=== DoidDAO 全面测试结束 ==="); + } + + @Before + public void clearData() { + // 每个测试前清空数据 + DoidDAO.clearAllStorage(); + } + + private static void cleanupTestDB() { + try { + Path dbPath = Paths.get(TEST_DB_PATH); + if (Files.exists(dbPath)) { + Files.walk(dbPath) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } catch (Exception e) { + System.err.println("清理测试数据库失败: " + e.getMessage()); + } + } + + // ==================== 基础功能测试 ==================== + + @Test + public void test01_BasicCRUD() throws Exception { + System.out.println("\n--- 基础 CRUD 操作测试 ---"); + + // 测试数据准备 + String doid = "test.doid.001"; + JsonObject testData = new JsonObject(); + testData.addProperty("type", "data"); + testData.addProperty("content", "test content"); + testData.addProperty("version", "1.0"); + + // 1. 测试插入 + DoidDAO.putDO(doid, testData); + System.out.println("✓ 插入操作成功"); + + // 2. 测试查询 + String result = DoidDAO.getDO(doid); + assertNotNull("查询结果不应为空", result); + + JsonObject retrieved = JsonParser.parseString(result).getAsJsonObject(); + assertEquals("data", retrieved.get("type").getAsString()); + assertEquals("test content", retrieved.get("content").getAsString()); + assertTrue(retrieved.has("_updateTimestamp")); + System.out.println("✓ 查询操作成功"); + + // 3. 测试更新 + testData.addProperty("content", "updated content"); + testData.addProperty("version", "2.0"); + DoidDAO.putDO(doid, testData); + + String updatedResult = DoidDAO.getDO(doid); + JsonObject updatedRetrieved = JsonParser.parseString(updatedResult).getAsJsonObject(); + assertEquals("updated content", updatedRetrieved.get("content").getAsString()); + assertEquals("2.0", updatedRetrieved.get("version").getAsString()); + System.out.println("✓ 更新操作成功"); + + // 4. 测试删除 + DoidDAO.deleteDO(doid); + + String deletedResult = DoidDAO.getDO(doid); + assertNull("删除后查询结果应为空", deletedResult); + System.out.println("✓ 删除操作成功"); + } + + @Test + public void test02_MultipleDataTypes() throws Exception { + System.out.println("\n--- 多类型数据存储测试 ---"); + + Map testTypes = new HashMap<>(); + testTypes.put("data.001", "data"); + testTypes.put("prefix.001", "prefix"); + testTypes.put("system.001", "system"); + testTypes.put("systemRepo.001", "systemRepo"); + testTypes.put("code.001", "code"); + testTypes.put("codeJava.001", "codeJava"); + + // 插入不同类型的数据 + for (Map.Entry entry : testTypes.entrySet()) { + String doid = entry.getKey(); + String type = entry.getValue(); + JsonObject data = new JsonObject(); + data.addProperty("type", type); + data.addProperty("content", "content for " + type); + + DoidDAO.putDO(doid, data); + } + + // 验证所有数据都能正确查询 + for (Map.Entry entry : testTypes.entrySet()) { + String doid = entry.getKey(); + String type = entry.getValue(); + String result = DoidDAO.getDO(doid); + assertNotNull("类型 " + type + " 的数据应该存在", result); + + JsonObject retrieved = JsonParser.parseString(result).getAsJsonObject(); + assertEquals(type, retrieved.get("type").getAsString()); + } + + System.out.println("✓ 多类型数据存储测试通过"); + } + + @Test + public void test03_CountingSystem() throws Exception { + System.out.println("\n--- 计数系统测试 ---"); + + // 测试初始计数 + assertEquals(0, DoidDAO.getCount("newType")); + + // 插入数据并验证计数 + for (int i = 0; i < 5; i++) { + JsonObject data = new JsonObject(); + data.addProperty("type", "testCount"); + data.addProperty("index", i); + DoidDAO.putDO("count.test." + i, data); + } + + assertEquals(5, DoidDAO.getCount("testCount")); + System.out.println("✓ 插入计数正确: " + DoidDAO.getCount("testCount")); + + // 删除数据并验证计数 + DoidDAO.deleteDO("count.test.0"); + assertEquals(4, DoidDAO.getCount("testCount")); + System.out.println("✓ 删除计数正确: " + DoidDAO.getCount("testCount")); + + // 测试不同类型的计数 + JsonObject prefixData = new JsonObject(); + prefixData.addProperty("type", "prefix"); + prefixData.addProperty("content", "prefix content"); + DoidDAO.putDO("prefix.test", prefixData); + + assertEquals(1, DoidDAO.getCount("prefix")); + assertEquals(4, DoidDAO.getCount("testCount")); + + System.out.println("✓ 计数系统测试通过"); + } + + // ==================== 查询功能测试 ==================== + + @Test + public void test04_TimeIndexQueries() throws Exception { + System.out.println("\n--- 时间索引查询测试 ---"); + + long baseTime = System.currentTimeMillis(); + + // 插入测试数据(不同时间戳) + for (int i = 0; i < 10; i++) { + JsonObject data = new JsonObject(); + data.addProperty("type", "data"); + data.addProperty("content", "content " + i); + data.addProperty("_updateTimestamp", baseTime + i * 1000); + + DoidDAO.putDO("time.test." + i, data); + Thread.sleep(10); // 确保时间戳不同 + } + + // 测试时间范围查询(正序) + JsonObject result = DoidDAO.queryDataList(baseTime, baseTime + 5000, 0, 5, true); + assertTrue("查询结果不应为空", result.size() > 0); + System.out.println("✓ 时间范围查询(正序)成功,结果数量: " + result.size()); + + // 测试时间范围查询(倒序) + JsonObject resultDesc = DoidDAO.queryDataList(baseTime, baseTime + 5000, 0, 5, false); + assertTrue("查询结果不应为空", resultDesc.size() > 0); + System.out.println("✓ 时间范围查询(倒序)成功,结果数量: " + resultDesc.size()); + + // 测试分页查询 + JsonObject page1 = DoidDAO.queryDataList(baseTime, baseTime + 10000, 0, 3, true); + JsonObject page2 = DoidDAO.queryDataList(baseTime, baseTime + 10000, 3, 3, true); + + assertTrue("第一页应有数据", page1.size() > 0); + assertTrue("第二页应有数据", page2.size() > 0); + System.out.println("✓ 分页查询成功"); + + System.out.println("✓ 时间索引查询测试通过"); + } + + @Test + public void test05_TypeSpecificQueries() throws Exception { + System.out.println("\n--- 类型特定查询测试 ---"); + + long baseTime = System.currentTimeMillis(); + + // 插入不同类型的数据 + String[] types = {"prefix", "system", "systemRepo", "code", "codeJava"}; + for (int i = 0; i < types.length; i++) { + for (int j = 0; j < 3; j++) { + JsonObject data = new JsonObject(); + data.addProperty("type", types[i]); + data.addProperty("content", "content " + i + "_" + j); + data.addProperty("_updateTimestamp", baseTime + (i * 3 + j) * 1000); + + DoidDAO.putDO(types[i] + ".test." + j, data); + } + } + + // 测试前缀查询 + JsonObject prefixResult = DoidDAO.queryPrefixList(baseTime, baseTime + 20000, 0, 10, true); + assertTrue("前缀查询应有结果", prefixResult.size() > 0); + System.out.println("✓ 前缀查询成功,结果数量: " + prefixResult.size()); + + // 测试系统类型查询 + JsonObject systemResult = DoidDAO.queryRepoList("system", baseTime, baseTime + 20000, 0, 10, true); + assertTrue("系统类型查询应有结果", systemResult.size() > 0); + System.out.println("✓ 系统类型查询成功,结果数量: " + systemResult.size()); + + // 测试代码类型查询 + JsonObject codeResult = DoidDAO.queryCodeList("code", baseTime, baseTime + 20000, 0, 10, true); + assertTrue("代码类型查询应有结果", codeResult.size() > 0); + System.out.println("✓ 代码类型查询成功,结果数量: " + codeResult.size()); + + System.out.println("✓ 类型特定查询测试通过"); + } + + @Test + public void test06_BatchOperations() throws Exception { + System.out.println("\n--- 批量操作测试 ---"); + + // 准备批量数据 + List doids = new ArrayList<>(); + for (int i = 0; i < SMALL_BATCH_SIZE; i++) { + String doid = "batch.test." + i; + doids.add(doid); + + JsonObject data = new JsonObject(); + data.addProperty("type", "data"); + data.addProperty("content", "batch content " + i); + data.addProperty("index", i); + + DoidDAO.putDO(doid, data); + } + + // 测试批量获取 + List results = DoidDAO.batchGetDO(doids); + assertEquals("批量获取结果数量应匹配", SMALL_BATCH_SIZE, results.size()); + + int validResults = 0; + for (byte[] result : results) { + if (result != null) { + validResults++; + JsonObject data = JsonParser.parseString(new String(result)).getAsJsonObject(); + assertEquals("data", data.get("type").getAsString()); + assertTrue(data.has("index")); + } + } + + assertEquals("所有批量查询结果都应有效", SMALL_BATCH_SIZE, validResults); + System.out.println("✓ 批量获取成功,有效结果: " + validResults); + + System.out.println("✓ 批量操作测试通过"); + } + + // ==================== 辅助表功能测试 ==================== + + @Test + public void test07_NamePubkeyOperations() throws Exception { + System.out.println("\n--- 名称公钥表操作测试 ---"); + + // 测试插入 + String key = "test.name.001"; + String doid = "doid.for.name.001"; + DoidDAO.putNamePubkey(key, doid); + System.out.println("✓ 名称公钥插入成功"); + + // 测试查询 + String result = DoidDAO.getNamePubkey(key); + assertEquals("查询结果应匹配", doid, result); + System.out.println("✓ 名称公钥查询成功"); + + // 测试不存在的键 + String nonExistent = DoidDAO.getNamePubkey("non.existent.key"); + assertNull("不存在的键应返回null", nonExistent); + + // 测试列表功能 + DoidDAO.putNamePubkey("test.name.002", "doid.for.name.002"); + DoidDAO.putNamePubkey("test.name.003", "doid.for.name.003"); + + JsonObject listResult = DoidDAO.listNamePubkey(); + assertTrue("列表结果应包含数据", listResult.size() >= 3); + System.out.println("✓ 名称公钥列表查询成功,数量: " + listResult.size()); + + // 测试删除 + DoidDAO.deleteNamePubkey(key); + String deletedResult = DoidDAO.getNamePubkey(key); + assertNull("删除后查询应返回null", deletedResult); + System.out.println("✓ 名称公钥删除成功"); + + System.out.println("✓ 名称公钥表操作测试通过"); + } + + @Test + public void test08_ListOperations() throws Exception { + System.out.println("\n--- 列表操作测试 ---"); + + // 准备测试数据 + for (int i = 0; i < 5; i++) { + JsonObject prefixData = new JsonObject(); + prefixData.addProperty("type", "prefix"); + prefixData.addProperty("content", "prefix content " + i); + DoidDAO.putDO("prefix.list." + i, prefixData); + + JsonObject repoData = new JsonObject(); + repoData.addProperty("type", "system"); + repoData.addProperty("content", "system content " + i); + DoidDAO.putDO("system.list." + i, repoData); + } + + // 测试前缀列表 + JsonObject prefixList = DoidDAO.listPrefix(); + assertEquals("success", prefixList.get("result").getAsString()); + assertTrue("前缀列表应有数据", prefixList.get("count").getAsInt() >= 5); + assertTrue("前缀列表应包含data字段", prefixList.has("data")); + System.out.println("✓ 前缀列表查询成功,数量: " + prefixList.get("count").getAsInt()); + + // 测试仓库列表 + JsonObject repoList = DoidDAO.listRepo(); + assertEquals("success", repoList.get("result").getAsString()); + assertTrue("仓库列表应有数据", repoList.get("count").getAsInt() >= 5); + assertTrue("仓库列表应包含data字段", repoList.has("data")); + System.out.println("✓ 仓库列表查询成功,数量: " + repoList.get("count").getAsInt()); + + System.out.println("✓ 列表操作测试通过"); + } + + // ==================== 兼容性和边界测试 ==================== + + @Test + public void test09_CompatibilityQuery() throws Exception { + System.out.println("\n--- 兼容性查询测试 ---"); + + long baseTime = System.currentTimeMillis(); + + // 插入测试数据 + String[] types = {"data", "prefix", "system", "code"}; + for (String type : types) { + JsonObject data = new JsonObject(); + data.addProperty("type", type); + data.addProperty("content", "content for " + type); + data.addProperty("_updateTimestamp", baseTime); + + DoidDAO.putDO(type + ".compat.test", data); + } + + // 测试通用查询方法 + JsonObject dataResult = DoidDAO.query("data", baseTime - 1000, baseTime + 1000, 0, 10, true); + assertTrue("data类型查询应有结果", dataResult.size() > 0); + + JsonObject prefixResult = DoidDAO.query("prefix", baseTime - 1000, baseTime + 1000, 0, 10, true); + assertTrue("prefix类型查询应有结果", prefixResult.size() > 0); + + JsonObject systemResult = DoidDAO.query("system", baseTime - 1000, baseTime + 1000, 0, 10, true); + assertTrue("system类型查询应有结果", systemResult.size() > 0); + + JsonObject codeResult = DoidDAO.query("code", baseTime - 1000, baseTime + 1000, 0, 10, true); + assertTrue("code类型查询应有结果", codeResult.size() > 0); + + // 测试空类型和null类型(应默认为data查询) + JsonObject nullResult = DoidDAO.query(null, baseTime - 1000, baseTime + 1000, 0, 10, true); + assertTrue("null类型查询应有结果", nullResult.size() > 0); + + JsonObject emptyResult = DoidDAO.query("", baseTime - 1000, baseTime + 1000, 0, 10, true); + assertTrue("空类型查询应有结果", emptyResult.size() > 0); + + System.out.println("✓ 兼容性查询测试通过"); + } + + @Test + public void test10_BoundaryConditions() throws Exception { + System.out.println("\n--- 边界条件测试 ---"); + + // 测试空查询 + JsonObject emptyResult = DoidDAO.queryDataList(0L, 1L, 0, 10, true); + assertNotNull("空查询结果不应为null", emptyResult); + + // 测试大偏移量 + JsonObject largeOffsetResult = DoidDAO.queryDataList(0L, Long.MAX_VALUE, 1000, 10, true); + assertNotNull("大偏移量查询结果不应为null", largeOffsetResult); + + // 测试零数量查询 + JsonObject zeroCountResult = DoidDAO.queryDataList(0L, Long.MAX_VALUE, 0, 0, true); + assertNotNull("零数量查询结果不应为null", zeroCountResult); + + // 测试不存在的DOID + String nonExistentResult = DoidDAO.getDO("non.existent.doid"); + assertNull("不存在的DOID应返回null", nonExistentResult); + + // 测试isExistedDoid + assertFalse("不存在的DOID检查应返回false", DoidDAO.isExistedDoid("non.existent.doid")); + + // 插入一个DOID并测试存在性 + JsonObject testData = new JsonObject(); + testData.addProperty("type", "data"); + testData.addProperty("content", "test"); + DoidDAO.putDO("exist.test", testData); + + assertTrue("存在的DOID检查应返回true", DoidDAO.isExistedDoid("exist.test")); + + System.out.println("✓ 边界条件测试通过"); + } + + // ==================== 性能测试 ==================== + + @Test + public void test11_PerformanceTest() throws Exception { + System.out.println("\n--- 性能测试 ---"); + + // 批量插入性能测试 + long startTime = System.currentTimeMillis(); + for (int i = 0; i < MEDIUM_BATCH_SIZE; i++) { + JsonObject data = new JsonObject(); + data.addProperty("type", "data"); + data.addProperty("content", "performance test content " + i); + data.addProperty("index", i); + + DoidDAO.putDO("perf.test." + i, data); + } + long insertTime = System.currentTimeMillis() - startTime; + System.out.println("✓ 批量插入 " + MEDIUM_BATCH_SIZE + " 条记录耗时: " + insertTime + "ms"); + + // 批量查询性能测试 + startTime = System.currentTimeMillis(); + List doids = new ArrayList<>(); + for (int i = 0; i < MEDIUM_BATCH_SIZE; i++) { + doids.add("perf.test." + i); + } + List results = DoidDAO.batchGetDO(doids); + long batchQueryTime = System.currentTimeMillis() - startTime; + System.out.println("✓ 批量查询 " + MEDIUM_BATCH_SIZE + " 条记录耗时: " + batchQueryTime + "ms"); + + // 单个查询性能测试 + startTime = System.currentTimeMillis(); + for (int i = 0; i < 100; i++) { + DoidDAO.getDO("perf.test." + i); + } + long singleQueryTime = System.currentTimeMillis() - startTime; + System.out.println("✓ 单个查询 100 条记录耗时: " + singleQueryTime + "ms"); + + // 时间范围查询性能测试 + startTime = System.currentTimeMillis(); + JsonObject timeRangeResult = DoidDAO.queryDataList(0L, Long.MAX_VALUE, 0, 100, true); + long timeRangeQueryTime = System.currentTimeMillis() - startTime; + System.out.println("✓ 时间范围查询耗时: " + timeRangeQueryTime + "ms,结果数量: " + timeRangeResult.size()); + + System.out.println("✓ 性能测试完成"); + } + + // ==================== 并发测试 ==================== + + @Test + public void test12_ConcurrencyTest() throws Exception { + System.out.println("\n--- 并发测试 ---"); + + int threadCount = 10; + int operationsPerThread = 20; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + List> futures = new ArrayList<>(); + + // 并发插入测试 + for (int t = 0; t < threadCount; t++) { + final int threadId = t; + Future future = executor.submit(() -> { + try { + for (int i = 0; i < operationsPerThread; i++) { + JsonObject data = new JsonObject(); + data.addProperty("type", "data"); + data.addProperty("content", "concurrent test " + threadId + "_" + i); + data.addProperty("threadId", threadId); + data.addProperty("index", i); + + DoidDAO.putDO("concurrent.test." + threadId + "." + i, data); + } + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } finally { + latch.countDown(); + } + }); + futures.add(future); + } + + // 等待所有线程完成 + latch.await(30, TimeUnit.SECONDS); + + // 检查所有操作是否成功 + boolean allSuccess = true; + for (Future future : futures) { + if (!future.get()) { + allSuccess = false; + break; + } + } + + assertTrue("所有并发操作应该成功", allSuccess); + + // 验证数据完整性 + int expectedCount = threadCount * operationsPerThread; + int actualCount = 0; + for (int t = 0; t < threadCount; t++) { + for (int i = 0; i < operationsPerThread; i++) { + String result = DoidDAO.getDO("concurrent.test." + t + "." + i); + if (result != null) { + actualCount++; + } + } + } + + assertEquals("并发插入的数据应该完整", expectedCount, actualCount); + System.out.println("✓ 并发插入测试通过,成功插入: " + actualCount + " 条记录"); + + executor.shutdown(); + System.out.println("✓ 并发测试完成"); + } + + // ==================== 数据库管理测试 ==================== + + @Test + public void test13_DatabaseManagement() throws Exception { + System.out.println("\n--- 数据库管理测试 ---"); + + // 插入一些测试数据 + for (int i = 0; i < 10; i++) { + JsonObject data = new JsonObject(); + data.addProperty("type", "data"); + data.addProperty("content", "management test " + i); + DoidDAO.putDO("mgmt.test." + i, data); + } + + // 验证数据存在 + String beforeClear = DoidDAO.getDO("mgmt.test.0"); + assertNotNull("清理前数据应该存在", beforeClear); + + // 测试清空所有存储 + Object clearResult = DoidDAO.clearAllStorage(); + assertNotNull("清空操作应该返回结果", clearResult); + + if (clearResult instanceof JsonObject) { + JsonObject result = (JsonObject) clearResult; + assertEquals("success", result.get("result").getAsString()); + } + + // 验证数据已清空 + String afterClear = DoidDAO.getDO("mgmt.test.0"); + assertNull("清理后数据应该不存在", afterClear); + + // 验证计数已重置 + assertEquals(0, DoidDAO.getCount("data")); + + System.out.println("✓ 数据库清空测试通过"); + + // 测试数据库统计信息 + String stats = DoidDAO.getDBStats(); + assertNotNull("数据库统计信息不应为null", stats); + assertTrue("统计信息应包含内容", stats.length() > 0); + System.out.println("✓ 数据库统计信息获取成功"); + + System.out.println("✓ 数据库管理测试通过"); + } + + // ==================== 数据一致性测试 ==================== + + @Test + public void test14_DataConsistency() throws Exception { + System.out.println("\n--- 数据一致性测试 ---"); + + // 测试不同类型数据的存储一致性 + Map typeToTable = new HashMap<>(); + typeToTable.put("data", "doidCF"); + typeToTable.put("prefix", "prefixCF"); + typeToTable.put("system", "repoCF"); + typeToTable.put("code", "doidCF"); + + for (Map.Entry entry : typeToTable.entrySet()) { + String type = entry.getKey(); + + // 插入数据 + JsonObject data = new JsonObject(); + data.addProperty("type", type); + data.addProperty("content", "consistency test for " + type); + String doid = "consistency.test." + type; + + DoidDAO.putDO(doid, data); + + // 验证数据可以查询到 + String result = DoidDAO.getDO(doid); + assertNotNull("类型 " + type + " 的数据应该可以查询到", result); + + JsonObject retrieved = JsonParser.parseString(result).getAsJsonObject(); + assertEquals(type, retrieved.get("type").getAsString()); + + // 验证计数正确 + assertTrue("类型 " + type + " 的计数应该大于0", DoidDAO.getCount(type) > 0); + } + + System.out.println("✓ 数据一致性测试通过"); + } + + // ==================== 错误处理测试 ==================== + + @Test + public void test15_ErrorHandling() throws Exception { + System.out.println("\n--- 错误处理测试 ---"); + + // 测试删除不存在的数据 + try { + DoidDAO.deleteDO("non.existent.doid"); + // 删除不存在的数据应该不抛出异常 + System.out.println("✓ 删除不存在数据处理正常"); + } catch (Exception e) { + // 如果抛出异常,也是可以接受的 + System.out.println("✓ 删除不存在数据抛出异常: " + e.getMessage()); + } + + // 测试空批量查询 + List emptyList = new ArrayList<>(); + List emptyResults = DoidDAO.batchGetDO(emptyList); + assertNotNull("空批量查询结果不应为null", emptyResults); + assertEquals("空批量查询结果应为空列表", 0, emptyResults.size()); + + // 测试无效时间范围查询 + JsonObject invalidTimeResult = DoidDAO.queryDataList(Long.MAX_VALUE, 0L, 0, 10, true); + assertNotNull("无效时间范围查询结果不应为null", invalidTimeResult); + + System.out.println("✓ 错误处理测试通过"); + } + + // ==================== 综合集成测试 ==================== + + @Test + public void test16_IntegrationTest() throws Exception { + System.out.println("\n--- 综合集成测试 ---"); + + long baseTime = System.currentTimeMillis(); + + // 模拟真实使用场景:插入各种类型的数据 + String[] types = {"data", "prefix", "system", "systemRepo", "code", "codeJava"}; + Map typeCounts = new HashMap<>(); + + for (String type : types) { + typeCounts.put(type, 0); + for (int i = 0; i < 5; i++) { + JsonObject data = new JsonObject(); + data.addProperty("type", type); + data.addProperty("content", "integration test " + type + " " + i); + data.addProperty("timestamp", baseTime + i * 1000); + data.addProperty("index", i); + + String doid = "integration." + type + "." + i; + DoidDAO.putDO(doid, data); + typeCounts.put(type, typeCounts.get(type) + 1); + } + } + + // 验证计数正确性 + for (Map.Entry entry : typeCounts.entrySet()) { + String type = entry.getKey(); + int expectedCount = entry.getValue(); + int actualCount = DoidDAO.getCount(type); + assertEquals("类型 " + type + " 的计数应该正确", expectedCount, actualCount); + } + + // 测试各种查询功能 + JsonObject dataQuery = DoidDAO.queryDataList(baseTime, baseTime + 10000, 0, 10, true); + assertTrue("数据查询应有结果", dataQuery.size() > 0); + + JsonObject prefixQuery = DoidDAO.queryPrefixList(baseTime, baseTime + 10000, 0, 10, true); + assertTrue("前缀查询应有结果", prefixQuery.size() > 0); + + // 测试名称公钥功能 + DoidDAO.putNamePubkey("integration.test.key", "integration.test.doid"); + String nameResult = DoidDAO.getNamePubkey("integration.test.key"); + assertEquals("integration.test.doid", nameResult); + + // 测试批量操作 + List batchDoids = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + batchDoids.add("integration.data." + i); + } + List batchResults = DoidDAO.batchGetDO(batchDoids); + assertEquals("批量查询结果数量应正确", 3, batchResults.size()); + + // 测试删除和计数更新 + DoidDAO.deleteDO("integration.data.0"); + int newDataCount = DoidDAO.getCount("data"); + assertEquals("删除后计数应减少", typeCounts.get("data") - 1, newDataCount); + + System.out.println("✓ 综合集成测试通过"); + } +} \ No newline at end of file diff --git a/yjs/GlobalRouter.yjs b/yjs/GlobalRouter.yjs index a5e31aa9be9fdc333d4565d3c1e9ae16dcd84340..7b6e0dd76f9a2c97a981b70d45f7f0f4da7f5727 100644 --- a/yjs/GlobalRouter.yjs +++ b/yjs/GlobalRouter.yjs @@ -1,4 +1,4 @@ -import "actemplate/ABAC.yjs"; +import "actemplate/RBAC.yjs"; import "routerModule.yjs"; import "testfuns.yjs"; @Permission("Async") @@ -10,7 +10,7 @@ oracle GlobalRouter { function onCreate(arg) { Global.isGlobal = true; Global.rootPrefix = ""; - initABAC(requester); + initRBAC(requester,Global.Resources.loadAsString("/role.json")); AsyncUtil.postFunction(initAsync, arg); } diff --git a/yjs/Router.yjs b/yjs/Router.yjs index 8b37ce346202fdd58b5d90294591bc6131c51c86..fc0f0399693e14a20d3c66641f215497a59550bc 100644 --- a/yjs/Router.yjs +++ b/yjs/Router.yjs @@ -1,4 +1,4 @@ -import "actemplate/ABAC.yjs"; +import "actemplate/RBAC.yjs"; import "routerModule.yjs"; @Permission("Async") @Permission("RocksDB") @@ -7,7 +7,7 @@ oracle Router { function onCreate(arg) { Global.isGlobal = false; Global.rootPrefix = ""; - initABAC(requester); + initRBAC(requester,Global.Resources.loadAsString("/role.json")); AsyncUtil.postFunction(initAsync, arg); } function initAsync(arg) { diff --git a/yjs/role.json b/yjs/role.json new file mode 100644 index 0000000000000000000000000000000000000000..e6b56bdfde52e415189df6b2307f785cf36769a9 --- /dev/null +++ b/yjs/role.json @@ -0,0 +1,5 @@ +{ + "userAdmin": ["addRole","removeRole"], + "authorizedUser": ["hello"], + "authorizedChildren": ["getLRSStatistics","storageLRSStatistics","deleteLRSStatistics"] +} \ No newline at end of file diff --git a/yjs/routerModule.yjs b/yjs/routerModule.yjs index 6a21457491381f63a25d372ed6d361af31d1c464..82477e4768a749f7179747feed3e2573e2e72c6c 100644 --- a/yjs/routerModule.yjs +++ b/yjs/routerModule.yjs @@ -44,9 +44,17 @@ module routerModule { return Math.abs(hash); } + @Description("参数为 {\"offset\":\"\",\"count\":\"\",\"doId\":\"\",\"startTs\":\"\",\"endTs\":\"\",\"order\":\"\"}传入doid,指获取一个唯一的标识信息") export function listLRS(arg) { - return org.bdware.datanet.router.Router.listLRS(); + if (!arg) { + arg = {} + }else{ + arg = JSON.parse(arg) + } + return org.bdware.datanet.router.Router.listLRS(arg); } + + @Description("参数为 {\"doId\":\"bdtest.china-mobile\",\"name\":\"中国移动\",\"pubKey\":\"123\"}") @ArgSchema({}) export function createLRS(arg) { return org.bdware.datanet.router.Router.createLRS(arg); @@ -64,9 +72,16 @@ module routerModule { return org.bdware.datanet.router.Router.getLRS(arg); } + @Description("参数为 {\"offset\":\"\",\"count\":\"\",\"doId\":\"\",\"startTs\":\"\",\"endTs\":\"\",\"type\":\"\",\"order\":\"\"}传入doid,指获取一个唯一的标识信息") export function listRepo(arg) { - return org.bdware.datanet.router.Router.listRepo(); + if (!arg) { + arg = {} + }else{ + arg = JSON.parse(arg) + } + return org.bdware.datanet.router.Router.listRepo(arg); } + @Description("参数为 {\"doId\":\"Registry\",\"name\":\"Registry\",\"pubKey\":\"122\"} ") @ArgSchema({}) export function createRepo(arg) { return org.bdware.datanet.router.Router.createRepo(arg); @@ -84,6 +99,19 @@ module routerModule { return org.bdware.datanet.router.Router.getRepo(arg); } @ArgSchema({}) + export function getLRSStatistics(arg){ + return org.bdware.datanet.router.Router.getLRSStatistics(arg); + } + @ArgSchema({}) + export function deleteLRSStatistics(arg){ + return org.bdware.datanet.router.Router.deleteLRSStatistics(arg); + } + @ArgSchema({}) + export function storageLRSStatistics(arg){ + return org.bdware.datanet.router.Router.storageLRSStatistics(arg); + } + @Description("参数为 {\"offset\":\"\",\"count\":\"\",\"doId\":\"\",\"startTs\":\"\",\"endTs\":\"\",\"order\":\"\"}传入doid,指获取一个唯一的标识信息") + @ArgSchema({}) export function listID(arg) { return org.bdware.datanet.router.Router.listID(arg); }