变更

.idea/dataSources.xml

@@ -8,13 +8,6 @@
       <jdbc-url>jdbc:clickhouse://localhost:8123/</jdbc-url>
       <working-dir>$ProjectFileDir$</working-dir>
     </data-source>
-    <data-source source="LOCAL" name="运维测试环境" uuid="b152edc5-14e2-4e8f-8b23-b3bb14498a26">
-      <driver-ref>clickhouse</driver-ref>
-      <synchronize>true</synchronize>
-      <jdbc-driver>ru.yandex.clickhouse.ClickHouseDriver</jdbc-driver>
-      <jdbc-url>jdbc:clickhouse://39.108.185.243:18123</jdbc-url>
-      <working-dir>$ProjectFileDir$</working-dir>
-    </data-source>
     <data-source source="LOCAL" name="运维生产环境" uuid="73b0f6e4-be42-4655-a2a7-187c8c532525">
       <driver-ref>clickhouse</driver-ref>
       <synchronize>true</synchronize>
@@ -44,11 +37,33 @@
       <working-dir>$ProjectFileDir$</working-dir>
     </data-source>
     <data-source source="LOCAL" name="test-taos" uuid="b6f9f0b4-a47e-40af-9a46-478f661ebd70">
+      <driver-ref>java.sql.Driver</driver-ref>
       <synchronize>true</synchronize>
       <configured-by-url>true</configured-by-url>
       <jdbc-driver>com.taosdata.jdbc.rs.RestfulDriver</jdbc-driver>
       <jdbc-url>jdbc:TAOS-RS://120.79.155.65:6041/thinker_cloud_device_service</jdbc-url>
       <working-dir>$ProjectFileDir$</working-dir>
     </data-source>
+    <data-source source="LOCAL" name="富能Clickhouse" uuid="6d4e19be-ba44-456a-80fa-87191ea6a96b">
+      <driver-ref>clickhouse</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>ru.yandex.clickhouse.ClickHouseDriver</jdbc-driver>
+      <jdbc-url>jdbc:clickhouse://cc-wz9e0b0g1iemr7062.public.clickhouse.ads.aliyuncs.com:8123</jdbc-url>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+    <data-source source="LOCAL" name="※富能-预发布" uuid="68c46c78-e25d-4a13-ad35-a1b20dad0754">
+      <driver-ref>clickhouse</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>ru.yandex.clickhouse.ClickHouseDriver</jdbc-driver>
+      <jdbc-url>jdbc:clickhouse://120.79.47.28:8123</jdbc-url>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+    <data-source source="LOCAL" name="新科云-预发布" uuid="ebc2bcaf-2688-4adf-9e2e-b6e99e0ba2ba">
+      <driver-ref>clickhouse</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>ru.yandex.clickhouse.ClickHouseDriver</jdbc-driver>
+      <jdbc-url>jdbc:clickhouse://101.33.208.124:8123</jdbc-url>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
   </component>
 </project>
\ No newline at end of file

.idea/sqldialects.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/fox/预发布.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/mutations.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/thinker-cloud/DDL.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/thinker-cloud/gateway_mqtt_log查询.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/thinker-cloud/三赢/清除三赢数据.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/thinker-cloud/性能排查.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/thinker-cloud/测试sql.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/慢sql.sql" dialect="ClickHouse" />
     <file url="file://$PROJECT_DIR$/按分区查询及删除数据.sql" dialect="ClickHouse" />
     <file url="file://$PROJECT_DIR$/按分区查询及删除系统表数据.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/查询计划.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/系统表TTL.sql" dialect="ClickHouse" />
     <file url="file://$PROJECT_DIR$/统计磁盘.sql" dialect="ClickHouse" />
     <file url="file://$PROJECT_DIR$/../fox/ddl.sql" dialect="ClickHouse" />
     <file url="file://$PROJECT_DIR$/../fox/pgToClickhouse.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/test_CollapsingMergeTree.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/test_ReplaceMergeTree.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/数据迁移项目/mutations.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/数据迁移项目/selectMeterData.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/数据迁移项目/分区.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/数据迁移项目/数据迁移过程.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/数据迁移项目/物化视图.sql" dialect="ClickHouse" />
+    <file url="file://$PROJECT_DIR$/../fox/数据迁移项目/统计磁盘.sql" dialect="ClickHouse" />
     <file url="file://$PROJECT_DIR$/../fox/查询轮询数据.sql" dialect="ClickHouse" />
   </component>
 </project>
\ No newline at end of file

fox/预发布.sql

+
+ALTER TABLE system.query_thread_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.query_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.trace_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.asynchronous_metric_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.session_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.part_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.metric_log MODIFY TTL event_date + toIntervalDay(1);
+
+truncate  table  system.query_log;
\ No newline at end of file

mutations.sql

@@ -9,7 +9,6 @@ KILL QUERY WHERE user='thinker'
 select query_id, read_rows, total_rows_approx, memory_usage, initial_user, initial_address, elapsed
     , query
     from system. processes _all order by read_rows desc;
-
 -- 37366
 SELECT
     count(1)

thinker-cloud/DDL.sql

+create database thinker_cloud_device_service;
+use thinker_cloud_device_service;
+-- 创建 device_polling 表
+create table IF NOT EXISTS device_polling
+(
+    id                  Int64,
+    raw_data_id         Int64,
+    timestamp           DateTime64(6),
+    gateway_id          Int64,
+    gateway_code        String,
+    polling_id          String,
+    type                Int32,
+    content             String,
+    last_frame_interval Int32
+) ENGINE = MergeTree
+      PARTITION BY toDate(timestamp)
+      ORDER BY (timestamp, gateway_id,polling_id);
+
+
+-- 创建 meter_data 表
+create table IF NOT EXISTS meter_data
+(
+    id            Int64,
+    value         String,
+    meter_attr_id Int64,
+    timestamp     DateTime64(6),
+    dirty         UInt8
+) ENGINE = ReplacingMergeTree(id)
+      PARTITION BY toDate(timestamp)
+      ORDER BY (timestamp, meter_attr_id);
+
+-- 创建 meter_analysis 表
+create table IF NOT EXISTS meter_analysis
+(
+    id            Int64,
+    meter_attr_id Int64,
+    value         String,
+    original      UInt8,
+    type          String,
+    timestamp     DateTime64(6)
+) ENGINE = MergeTree
+      PARTITION BY toDate(timestamp)
+      ORDER BY (timestamp, meter_attr_id);
+
+-- 创建 meter_event_log 表
+-- create table IF NOT EXISTS meter_event_log
+-- (
+--     id             Int64,
+--     event_id       Int64,
+--     event_value    String,
+--     description    String,
+--     event_name     String,
+--     create_time    DateTime64(6),
+--     gateway_id     Int64,
+--     polling_id     String,
+--     raw_data_id    Int64,
+--     original_value String,
+--     custom_is      UInt8,
+--     last_value     String,
+--     change_is      UInt8
+-- ) ENGINE = MergeTree
+--       PARTITION BY toDate(create_time)
+--       ORDER BY create_time;
+
+-- 创建 reporting_raw_data 表
+create table IF NOT EXISTS reporting_raw_data
+(
+    id          Int64,
+    device_code String,
+    payload     String,
+    timestamp   DateTime64(6)
+) ENGINE = MergeTree
+      PARTITION BY toDate(timestamp)
+      ORDER BY (timestamp, device_code);
+
+-- 创建 meter_data_storage_extend 表
+create table IF NOT EXISTS meter_data_storage_extend
+(
+    data_id                  Int64,
+    functions         String,
+    meter_attr_id          Int64,
+    timestamp           DateTime64(6),
+    value        String
+) ENGINE = MergeTree
+      PARTITION BY toMonday(timestamp)
+      ORDER BY (timestamp, meter_attr_id);
+
+
+-- 创建 gateway_log 表
+create table IF NOT EXISTS gateway_log
+(
+    gateway_code                  String,
+    level         String,
+    offline_is         UInt8,
+    content          String,
+    timestamp           DateTime64(3)
+) ENGINE = MergeTree
+      PARTITION BY toMonth(timestamp)
+      ORDER BY (timestamp,gateway_code,level, offline_is);
+
+
+-- 创建 gateway_mqtt_log 表
+create table gateway_mqtt_log
+(
+    gateway_code String,
+    topic        String,
+    content      String,
+    timestamp    DateTime64(3)
+)
+    engine = MergeTree
+        PARTITION BY toMonth(timestamp)
+        ORDER BY (timestamp, gateway_code, topic);
+
+-- device_log_info
+create table device_log_info
+(
+    id               Int64,
+    device_id        Int64,
+    device_name      String,
+    device_type_id   Int64,
+    state            Int32,
+    content          String,
+    account          String,
+    ip               String,
+    business_type_id Int32,
+    business_id      Int64,
+    project_id       Int64,
+    create_time      DateTime64(6),
+    extended_info    String,
+    account_id       Int64
+)
+    engine = MergeTree PARTITION BY toWeek(create_time)
+        ORDER BY (create_time, device_id);
+-- error_meter_data
+CREATE TABLE error_meter_data
+(
+
+    `id` Int64,
+
+    `description` String,
+
+    `markers` String,
+
+    `old_value` String,
+
+    `new_value` String,
+
+    `meter_attr_id` Int64,
+
+    `timestamp` DateTime64(6)
+)
+ENGINE = MergeTree
+PARTITION BY toDate(timestamp)
+ORDER BY (timestamp,
+ meter_attr_id)
+SETTINGS index_granularity = 8192;
+
+
+
+
+-- 创建 meter_event_log 表
+create table IF NOT EXISTS meter_event_log
+(
+    id             Int64,
+    event_id       Int64,
+    create_time    DateTime64(6),
+    event_value    String,
+    original_value String,
+    last_value     String,
+    change_is      UInt8,
+    event_value_desc String
+) ENGINE = MergeTree
+      PARTITION BY toDate(create_time)
+      ORDER BY (create_time, event_id);
\ No newline at end of file

thinker-cloud/gateway_mqtt_log查询.sql

+
+select count(1),gateway_code
+from gateway_mqtt_log
+where timestamp >='2023-05-27 00:00:00' and timestamp <='2023-05-27 23:59:59'
+group by gateway_code
+order by count(1) desc;
+
+select count(1),gateway_code
+from gateway_mqtt_log
+where timestamp >='2023-04-14 00:00:00' and timestamp <='2023-04-15 00:00:00'
+group by gateway_code
+order by count(1) desc;
+
+
+select count(1),topic
+from gateway_mqtt_log
+where timestamp >='2023-04-10 00:00:00' and timestamp <='2023-04-11 00:00:00' and gateway_code='A2211090046'
+group by topic
+order by count(1) desc;
\ No newline at end of file

thinker-cloud/性能排查.sql

+show processlist;
+
+select *
+from system.processes;
+
+
+select query_start_time,query_duration_ms,query
+from system.query_log
+where event_time>=now()-interval 8 minute and query_duration_ms>500
+order by query_duration_ms desc
+limit 100;
+
+select *
+from system.query_log
+where event_time>=now()-interval 10 minute and query_duration_ms>1000
+order by query_duration_ms desc
+limit 100;
+
+-- 数据存储频率
+select count(1)
+from device_polling
+where timestamp>=now() -interval 10 MINUTE;
+
+select count(1)
+from meter_data
+where timestamp>=now() -interval 10 MINUTE;
+
+
+select meter_attr_id,count(1)
+from meter_data
+where timestamp >='2023-03-24 13:41:00' and timestamp <='2023-03-24 13:51:00'
+group by meter_attr_id;
+
+
+select meter_attr_id,count(1)
+    ,argMin(timestamp, timestamp)
+from meter_data
+where timestamp >='2023-03-24 13:30:00' and timestamp <='2023-03-24 13:58:00'
+group by meter_attr_id
+having  count(1)>10
+order by argMin(timestamp, timestamp) desc
+
+
+
+select *
+from meter_data
+where timestamp >='2023-03-24 13:30:00' and timestamp <='2023-03-24 13:58:00' and meter_attr_id=73693
+order by timestamp
+
+
+
+
+
+select count(1)
+from device_polling
+where timestamp >='2023-04-01 00:00:00' and timestamp <='2023-04-02 00:00:00';
+select count(1)
+from device_polling
+where timestamp >='2023-04-10 00:00:00' and timestamp <='2023-04-11 00:00:00';
+
+select count(1)
+from meter_data
+where timestamp >='2023-04-01 00:00:00' and timestamp <='2023-04-02 00:00:00';
+select count(1)
+from meter_data
+where timestamp >='2023-04-08 00:00:00' and timestamp <='2023-04-09 00:00:00';
\ No newline at end of file

thinker-cloud/测试sql.sql

+with
+        temp as (
+        select base.*
+        from meter_data as base
+        where timestamp >='2021-11-04 11:00:00' and timestamp <='2021-11-04 12:00:00' and meter_attr_id in(78038,28775)
+        order by timestamp desc
+        )
+        select temp.*
+        from temp
+        where id in(
+        select
+        max(temp.id)
+        from temp group by meter_attr_id
+        )
+
+
+select count(1)
+from gateway_mqtt_log
\ No newline at end of file

慢sql.sql

+SELECT
+    event_time,
+    query_log.query,
+    user,
+    query_id AS query,
+    read_rows,
+    read_bytes,
+    result_rows,
+    result_bytes,
+    memory_usage,
+    exception,
+       query_duration_ms
+FROM system.query_log
+WHERE  query_duration_ms > 50 AND (is_initial_query = 1) AND (query NOT LIKE 'INSERT INTO%')
+ORDER BY query_duration_ms desc
+LIMIT 100

按分区查询及删除系统表数据.sql

@@ -24,6 +24,12 @@ alter table system.query_log delete where event_date<'2022-06-30';
 alter table system.query_thread_log delete where event_date<'2022-06-30';
 
 
+alter table meter_data
+update value = '0.5'
+where meter_attr_id= 40093
+and timestamp >= '2024-01-05 01:49:59.998';
+
+-- 设置表TTL
 ALTER TABLE system.part_log MODIFY TTL event_date + toIntervalDay(2);
 
 truncate table system.session_log;

查询计划.sql

+explain indexes=1
+select *
+from meter_data
+where meter_data.timestamp>'2023-04-26 00:00:00'
+--   and meter_attr_id=13
+order by timestamp desc
+limit 1
+
+
+select count(*)
+from meter_data
+where meter_data.timestamp>'2022-04-26 00:00:00' and meter_attr_id in (78037,
+78038,
+78037,
+73359,
+63772,
+69860,
+69859,
+69858,
+69862,
+69861,
+69856,
+69855
+)
+order by timestamp desc
+limit 1
\ No newline at end of file

系统表TTL.sql

+
+ALTER TABLE system.query_thread_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.query_log MODIFY TTL event_date + toIntervalDay(1);
+ALTER TABLE system.trace_log MODIFY TTL event_date + toIntervalDay(10);
+ALTER TABLE system.asynchronous_metric_log MODIFY TTL event_date + toIntervalDay(10);
+ALTER TABLE system.session_log MODIFY TTL event_date + toIntervalDay(10);
+ALTER TABLE system.part_log MODIFY TTL event_date + toIntervalDay(10);
+ALTER TABLE system.metric_log MODIFY TTL event_date + toIntervalDay(30);
\ No newline at end of file

统计磁盘.sql

@@ -27,14 +27,16 @@ from
         size / (max_date - min_date) as avgDaySize
     from system.parts
     where active
-     and database = 'system'
---      and database = 'thinker_cloud_device_service'
+--      and database = 'system'
+     and database = 'thinker_cloud_device_service'
 --      and table = 'meter_data'
     group by
         database,
         table
     order by size desc
 );
+
+
 -- 数据库容量统计
 select
     sum(rows) as rows,--总行数
@@ -67,6 +69,6 @@ SELECT query_id,user, query FROM system.processes;
         GROUP BY xValueDateTime
         order by xValueDateTime;
 
+truncate table reporting_raw_data;
 
-ALTER TABLE system.parts MODIFY TTL event_date + toIntervalDay(10);
 select version()
\ No newline at end of file

跳树索引.sql

+
+-- 创建跳树索引
+ALTER TABLE meter_data ADD INDEX meter_data_attr_id_vix meter_attr_id TYPE bloom_filter GRANULARITY 1;
+
+-- 优化索引
+ALTER TABLE meter_data MATERIALIZE INDEX meter_data_attr_id_vix;
+
+-- 删除索引
+ALTER TABLE meter_data DROP INDEX  meter_data_attr_id_vix;
+
+
+-- 查看优化索引情况
+SELECT
+    mutation_id,
+    database,
+    table,
+    command,
+    create_time,
+    is_done
+FROM system.mutations
+where is_done = 0
\ No newline at end of file