[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill-6745358a-f659-4cdf-9c8a-ea6354db09d8":3,"$feJxfGoIqyC1Nh4c2-FSayJrPJMmFadk7j8_FxjF6ONE":42},{"id":4,"title":5,"description":6,"categoryId":7,"moduleId":8,"tags":9,"prompt":10,"icon":11,"source":12,"sourceUrl":13,"authorId":14,"authorName":15,"isPublic":16,"stars":17,"runs":18,"createdAt":19,"updatedAt":19,"module":20,"category":27,"packages":33},"6745358a-f659-4cdf-9c8a-ea6354db09d8","database-migrations-migration-observability","移民监控、疾病控制中心、可观察性基础设施","cat_coding_backend","mod_coding","sickn33,coding","---\nname: database-migrations-migration-observability\ndescription: \"Migration monitoring, CDC, and observability infrastructure\"\nrisk: unknown\nsource: community\ntags: \"database, cdc, debezium, kafka, prometheus, grafana, monitoring\"\ndate_added: \"2026-02-27\"\n---\n\n# Migration Observability and Real-time Monitoring\n\nYou are a database observability expert specializing in Change Data Capture, real-time migration monitoring, and enterprise-grade observability infrastructure. Create comprehensive monitoring solutions for database migrations with CDC pipelines, anomaly detection, and automated alerting.\n\n## Use this skill when\n\n- Working on migration observability and real-time monitoring tasks or workflows\n- Needing guidance, best practices, or checklists for migration observability and real-time monitoring\n\n## Do not use this skill when\n\n- The task is unrelated to migration observability and real-time monitoring\n- You need a different domain or tool outside this scope\n\n## Context\nThe user needs observability infrastructure for database migrations, including real-time data synchronization via CDC, comprehensive metrics collection, alerting systems, and visual dashboards.\n\n## Requirements\n$ARGUMENTS\n\n## Instructions\n\n### 1. Observable MongoDB Migrations\n\n```javascript\nconst { MongoClient } = require('mongodb');\nconst { createLogger, transports } = require('winston');\nconst prometheus = require('prom-client');\n\nclass ObservableAtlasMigration {\n    constructor(connectionString) {\n        this.client = new MongoClient(connectionString);\n        this.logger = createLogger({\n            transports: [\n                new transports.File({ filename: 'migrations.log' }),\n                new transports.Console()\n            ]\n        });\n        this.metrics = this.setupMetrics();\n    }\n\n    setupMetrics() {\n        const register = new prometheus.Registry();\n\n        return {\n            migrationDuration: new prometheus.Histogram({\n                name: 'mongodb_migration_duration_seconds',\n                help: 'Duration of MongoDB migrations',\n                labelNames: ['version', 'status'],\n                buckets: [1, 5, 15, 30, 60, 300],\n                registers: [register]\n            }),\n            documentsProcessed: new prometheus.Counter({\n                name: 'mongodb_migration_documents_total',\n                help: 'Total documents processed',\n                labelNames: ['version', 'collection'],\n                registers: [register]\n            }),\n            migrationErrors: new prometheus.Counter({\n                name: 'mongodb_migration_errors_total',\n                help: 'Total migration errors',\n                labelNames: ['version', 'error_type'],\n                registers: [register]\n            }),\n            register\n        };\n    }\n\n    async migrate() {\n        await this.client.connect();\n        const db = this.client.db();\n\n        for (const [version, migration] of this.migrations) {\n            await this.executeMigrationWithObservability(db, version, migration);\n        }\n    }\n\n    async executeMigrationWithObservability(db, version, migration) {\n        const timer = this.metrics.migrationDuration.startTimer({ version });\n        const session = this.client.startSession();\n\n        try {\n            this.logger.info(`Starting migration ${version}`);\n\n            await session.withTransaction(async () => {\n                await migration.up(db, session, (collection, count) => {\n                    this.metrics.documentsProcessed.inc({\n                        version,\n                        collection\n                    }, count);\n                });\n            });\n\n            timer({ status: 'success' });\n            this.logger.info(`Migration ${version} completed`);\n\n        } catch (error) {\n            this.metrics.migrationErrors.inc({\n                version,\n                error_type: error.name\n            });\n            timer({ status: 'failed' });\n            throw error;\n        } finally {\n            await session.endSession();\n        }\n    }\n}\n```\n\n### 2. Change Data Capture with Debezium\n\n```python\nimport asyncio\nimport json\nfrom kafka import KafkaConsumer, KafkaProducer\nfrom prometheus_client import Counter, Histogram, Gauge\nfrom datetime import datetime\n\nclass CDCObservabilityManager:\n    def __init__(self, config):\n        self.config = config\n        self.metrics = self.setup_metrics()\n\n    def setup_metrics(self):\n        return {\n            'events_processed': Counter(\n                'cdc_events_processed_total',\n                'Total CDC events processed',\n                ['source', 'table', 'operation']\n            ),\n            'consumer_lag': Gauge(\n                'cdc_consumer_lag_messages',\n                'Consumer lag in messages',\n                ['topic', 'partition']\n            ),\n            'replication_lag': Gauge(\n                'cdc_replication_lag_seconds',\n                'Replication lag',\n                ['source_table', 'target_table']\n            )\n        }\n\n    async def setup_cdc_pipeline(self):\n        self.consumer = KafkaConsumer(\n            'database.changes',\n            bootstrap_servers=self.config['kafka_brokers'],\n            group_id='migration-consumer',\n            value_deserializer=lambda m: json.loads(m.decode('utf-8'))\n        )\n\n        self.producer = KafkaProducer(\n            bootstrap_servers=self.config['kafka_brokers'],\n            value_serializer=lambda v: json.dumps(v).encode('utf-8')\n        )\n\n    async def process_cdc_events(self):\n        for message in self.consumer:\n            event = self.parse_cdc_event(message.value)\n\n            self.metrics['events_processed'].labels(\n                source=event.source_db,\n                table=event.table,\n                operation=event.operation\n            ).inc()\n\n            await self.apply_to_target(\n                event.table,\n                event.operation,\n                event.data,\n                event.timestamp\n            )\n\n    async def setup_debezium_connector(self, source_config):\n        connector_config = {\n            \"name\": f\"migration-connector-{source_config['name']}\",\n            \"config\": {\n                \"connector.class\": \"io.debezium.connector.postgresql.PostgresConnector\",\n                \"database.hostname\": source_config['host'],\n                \"database.port\": source_config['port'],\n                \"database.dbname\": source_config['database'],\n                \"plugin.name\": \"pgoutput\",\n                \"heartbeat.interval.ms\": \"10000\"\n            }\n        }\n\n        response = requests.post(\n            f\"{self.config['kafka_connect_url']}\u002Fconnectors\",\n            json=connector_config\n        )\n```\n\n### 3. Enterprise Monitoring and Alerting\n\n```python\nfrom prometheus_client import Counter, Gauge, Histogram, Summary\nimport numpy as np\n\nclass EnterpriseMigrationMonitor:\n    def __init__(self, config):\n        self.config = config\n        self.registry = prometheus.CollectorRegistry()\n        self.metrics = self.setup_metrics()\n        self.alerting = AlertingSystem(config.get('alerts', {}))\n\n    def setup_metrics(self):\n        return {\n            'migration_duration': Histogram(\n                'migration_duration_seconds',\n                'Migration duration',\n                ['migration_id'],\n                buckets=[60, 300, 600, 1800, 3600],\n                registry=self.registry\n            ),\n            'rows_migrated': Counter(\n                'migration_rows_total',\n                'Total rows migrated',\n                ['migration_id', 'table_name'],\n                registry=self.registry\n            ),\n            'data_lag': Gauge(\n                'migration_data_lag_seconds',\n                'Data lag',\n                ['migration_id'],\n                registry=self.registry\n            )\n        }\n\n    async def track_migration_progress(self, migration_id):\n        while migration.status == 'running':\n            stats = await self.calculate_progress_stats(migration)\n\n            self.metrics['rows_migrated'].labels(\n                migration_id=migration_id,\n                table_name=migration.table\n            ).inc(stats.rows_processed)\n\n            anomalies = await self.detect_anomalies(migration_id, stats)\n            if anomalies:\n                await self.handle_anomalies(migration_id, anomalies)\n\n            await asyncio.sleep(30)\n\n    async def detect_anomalies(self, migration_id, stats):\n        anomalies = []\n\n        if stats.rows_per_second \u003C stats.expected_rows_per_second * 0.5:\n            anomalies.append({\n                'type': 'low_throughput',\n                'severity': 'warning',\n                'message': f'Throughput below expected'\n            })\n\n        if stats.error_rate > 0.01:\n            anomalies.append({\n                'type': 'high_error_rate',\n                'severity': 'critical',\n                'message': f'Error rate exceeds threshold'\n            })\n\n        return anomalies\n\n    async def setup_migration_dashboard(self):\n        dashboard_config = {\n            \"dashboard\": {\n                \"title\": \"Database Migration Monitoring\",\n                \"panels\": [\n                    {\n                        \"title\": \"Migration Progress\",\n                        \"targets\": [{\n                            \"expr\": \"rate(migration_rows_total[5m])\"\n                        }]\n                    },\n                    {\n                        \"title\": \"Data Lag\",\n                        \"targets\": [{\n                            \"expr\": \"migration_data_lag_seconds\"\n                        }]\n                    }\n                ]\n            }\n        }\n\n        response = requests.post(\n            f\"{self.config['grafana_url']}\u002Fapi\u002Fdashboards\u002Fdb\",\n            json=dashboard_config,\n            headers={'Authorization': f\"Bearer {self.config['grafana_token']}\"}\n        )\n\nclass AlertingSystem:\n    def __init__(self, config):\n        self.config = config\n\n    async def send_alert(self, title, message, severity, **kwargs):\n        if 'slack' in self.config:\n            await self.send_slack_alert(title, message, severity)\n\n        if 'email' in self.config:\n            await self.send_email_alert(title, message, severity)\n\n    async def send_slack_alert(self, title, message, severity):\n        color = {\n            'critical': 'danger',\n            'warning': 'warning',\n            'info': 'good'\n        }.get(severity, 'warning')\n\n        payload = {\n            'text': title,\n            'attachments': [{\n                'color': color,\n                'text': message\n            }]\n        }\n\n        requests.post(self.config['slack']['webhook_url'], json=payload)\n```\n\n### 4. Grafana Dashboard Configuration\n\n```python\ndashboard_panels = [\n    {\n        \"id\": 1,\n        \"title\": \"Migration Progress\",\n        \"type\": \"graph\",\n        \"targets\": [{\n            \"expr\": \"rate(migration_rows_total[5m])\",\n            \"legendFormat\": \"{{migration_id}} - {{table_name}}\"\n        }]\n    },\n    {\n        \"id\": 2,\n        \"title\": \"Data Lag\",\n        \"type\": \"stat\",\n        \"targets\": [{\n            \"expr\": \"migration_data_lag_seconds\"\n        }],\n        \"fieldConfig\": {\n            \"thresholds\": {\n                \"steps\": [\n                    {\"value\": 0, \"color\": \"green\"},\n                    {\"value\": 60, \"color\": \"yellow\"},\n                    {\"value\": 300, \"color\": \"red\"}\n                ]\n            }\n        }\n    },\n    {\n        \"id\": 3,\n        \"title\": \"Error Rate\",\n        \"type\": \"graph\",\n        \"targets\": [{\n            \"expr\": \"rate(migration_errors_total[5m])\"\n        }]\n    }\n]\n```\n\n### 5. CI\u002FCD Integration\n\n```yaml\nname: Migration Monitoring\n\non:\n  push:\n    branches: [main]\n\njobs:\n  monitor-migration:\n    runs-on: ubuntu-latest\n\n    steps:\n      - uses: actions\u002Fcheckout@v4\n\n      - name: Start Monitoring\n        run: |\n          python migration_monitor.py start \\\n            --migration-id ${{ github.sha }} \\\n            --prometheus-url ${{ secrets.PROMETHEUS_URL }}\n\n      - name: Run Migration\n        run: |\n          python migrate.py --environment production\n\n      - name: Check Migration Health\n        run: |\n          python migration_monitor.py check \\\n            --migration-id ${{ github.sha }} \\\n            --max-lag 300\n```\n\n## Output Format\n\n1. **Observable MongoDB Migrations**: Atlas framework with metrics and validation\n2. **CDC Pipeline with Monitoring**: Debezium integration with Kafka\n3. **Enterprise Metrics Collection**: Prometheus instrumentation\n4. **Anomaly Detection**: Statistical analysis\n5. **Multi-channel Alerting**: Email, Slack, PagerDuty integrations\n6. **Grafana Dashboard Automation**: Programmatic dashboard creation\n7. **Replication Lag Tracking**: Source-to-target lag monitoring\n8. **Health Check Systems**: Continuous pipeline monitoring\n\nFocus on real-time visibility, proactive alerting, and comprehensive observability for zero-downtime migrations.\n\n## Cross-Plugin Integration\n\nThis plugin integrates with:\n- **sql-migrations**: Provides observability for SQL migrations\n- **nosql-migrations**: Monitors NoSQL transformations\n- **migration-integration**: Coordinates monitoring across workflows\n\n## Limitations\n- Use this skill only when the task clearly matches the scope described above.\n- Do not treat the output as a substitute for environment-specific validation, testing, or expert review.\n- Stop and ask for clarification if required inputs, permissions, safety boundaries, or success criteria are missing.\n","","imported","https:\u002F\u002Fgithub.com\u002Fsickn33\u002Fantigravity-awesome-skills","user_system_seed","SkillOPIC",true,148,1523,"2026-05-16 13:14:19",{"id":8,"name":21,"slug":22,"icon":23,"description":24,"sort":25,"createdAt":26},"编程开发","coding","mdi-code-braces","代码生成、调试、审查，提升开发效率",2,"2026-05-16 12:53:40",{"id":7,"name":28,"slug":29,"icon":30,"description":31,"moduleId":8,"sort":25,"skillCount":32,"createdAt":26},"后端开发","backend","mdi-server","API、数据库、服务端架构",296,[34],{"id":35,"skillId":4,"version":36,"fileName":37,"fileSize":38,"filePath":39,"fileHash":40,"manifest":41,"createdAt":19},"29216e52-bc35-4e67-8024-3ff7a1a9458d","1.0.0","database-migrations-migration-observability.zip",4098,"uploads\u002Fskills\u002F6745358a-f659-4cdf-9c8a-ea6354db09d8\u002Fdatabase-migrations-migration-observability.zip","1fe81ea00a00bd4a68f47fadf45d5214f68c423dc3b75b14f617564904a1203e","[{\"path\":\"SKILL.md\",\"isDirectory\":false,\"size\":13565}]",{"code":43,"message":44,"data":45},200,"success",{"items":46,"stats":47,"page":50},[],{"averageRating":48,"totalRatings":48,"ratingCounts":49},0,[48,48,48,48,48],{"limit":51,"offset":48,"hasMore":52,"nextOffset":51,"ratedOnly":16},15,false]