fix: double skills cleaning and script
This commit is contained in:
Julien Froidefond
4
.gitignore
vendored
4
.gitignore
vendored
@@ -86,3 +86,7 @@ pids
|
||||
# Temporary folders
|
||||
tmp/
|
||||
temp/
|
||||
|
||||
# Scripts
|
||||
duplicates-report.txt
|
||||
cleaning-report.txt
|
||||
@@ -125,16 +125,7 @@
|
||||
"https://www.rabbitmq.com/documentation.html"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "apache-kafka",
|
||||
"name": "Apache Kafka",
|
||||
"description": "Plateforme de streaming distribuée",
|
||||
"icon": "fas-broadcast",
|
||||
"links": [
|
||||
"https://kafka.apache.org/",
|
||||
"https://kafka.apache.org/documentation/"
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "semantic-kernel",
|
||||
"name": "Semantic Kernel",
|
||||
|
||||
@@ -124,23 +124,7 @@
|
||||
],
|
||||
"icon": "fab-microsoft"
|
||||
},
|
||||
{
|
||||
"id": "mongodb",
|
||||
"name": "MongoDB",
|
||||
"description": "Base de données NoSQL orientée documents, offrant flexibilité et scalabilité pour les applications modernes",
|
||||
"links": ["https://www.mongodb.com/", "https://docs.mongodb.com/"],
|
||||
"icon": "fas-leaf"
|
||||
},
|
||||
{
|
||||
"id": "postgresql",
|
||||
"name": "PostgreSQL",
|
||||
"description": "Système de gestion de base de données relationnelle open source avec des fonctionnalités avancées et une grande extensibilité",
|
||||
"links": [
|
||||
"https://www.postgresql.org/",
|
||||
"https://www.postgresql.org/docs/"
|
||||
],
|
||||
"icon": "fas-database"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "snowflake",
|
||||
"name": "Snowflake",
|
||||
@@ -161,23 +145,6 @@
|
||||
"description": "Fork d'Elasticsearch, offrant des fonctionnalités de recherche et d'analyse de données distribuées",
|
||||
"links": ["https://opensearch.org/", "https://opensearch.org/docs/"],
|
||||
"icon": "fas-search"
|
||||
},
|
||||
{
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"description": "Plateforme open source de visualisation et d'analyse de données, particulièrement adaptée pour le monitoring et les métriques",
|
||||
"links": ["https://grafana.com/", "https://grafana.com/docs/"],
|
||||
"icon": "fas-chart-line"
|
||||
},
|
||||
{
|
||||
"id": "powerbi",
|
||||
"name": "Power BI",
|
||||
"description": "Plateforme de Business Intelligence de Microsoft pour l'analyse et la visualisation de données avec des tableaux de bord interactifs",
|
||||
"links": [
|
||||
"https://powerbi.microsoft.com/fr-fr/",
|
||||
"https://docs.microsoft.com/power-bi/"
|
||||
],
|
||||
"icon": "fab-microsoft"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -144,16 +144,7 @@
|
||||
"links": ["https://istio.io/", "https://istio.io/latest/docs/"],
|
||||
"icon": "fas-cog"
|
||||
},
|
||||
{
|
||||
"id": "vault",
|
||||
"name": "HashiCorp Vault",
|
||||
"description": "Gestion des secrets et chiffrement",
|
||||
"links": [
|
||||
"https://www.vaultproject.io/",
|
||||
"https://learn.hashicorp.com/vault"
|
||||
],
|
||||
"icon": "fas-cog"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "consul",
|
||||
"name": "HashiCorp Consul",
|
||||
@@ -178,16 +169,7 @@
|
||||
"links": ["https://konghq.com/", "https://docs.konghq.com/"],
|
||||
"icon": "fas-door-open"
|
||||
},
|
||||
{
|
||||
"id": "keycloak",
|
||||
"name": "Keycloak",
|
||||
"description": "Solution open source de gestion des identités et des accès (IAM) avec support de l'authentification unique (SSO)",
|
||||
"links": [
|
||||
"https://www.keycloak.org/",
|
||||
"https://www.keycloak.org/documentation"
|
||||
],
|
||||
"icon": "fas-key"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "ceph",
|
||||
"name": "Ceph",
|
||||
@@ -239,16 +221,7 @@
|
||||
],
|
||||
"icon": "fas-database"
|
||||
},
|
||||
{
|
||||
"id": "vault",
|
||||
"name": "HashiCorp Vault",
|
||||
"description": "Système de gestion des secrets et de protection des données sensibles avec chiffrement et rotation automatique",
|
||||
"links": [
|
||||
"https://www.vaultproject.io/",
|
||||
"https://learn.hashicorp.com/vault"
|
||||
],
|
||||
"icon": "fas-vault"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "sonar",
|
||||
"name": "Sonar",
|
||||
@@ -280,23 +253,7 @@
|
||||
"links": ["https://checkmarx.com/", "https://checkmarx.com/resources/"],
|
||||
"icon": "fas-shield-alt"
|
||||
},
|
||||
{
|
||||
"id": "stryker",
|
||||
"name": "Stryker",
|
||||
"description": "Suite d'outils de test par mutation pour évaluer la qualité des tests unitaires en modifiant le code source",
|
||||
"links": [
|
||||
"https://stryker-mutator.io/",
|
||||
"https://stryker-mutator.io/docs/"
|
||||
],
|
||||
"icon": "fas-bug"
|
||||
},
|
||||
{
|
||||
"id": "helm",
|
||||
"name": "Helm",
|
||||
"description": "Gestionnaire de packages pour Kubernetes facilitant le déploiement et la configuration des applications",
|
||||
"links": ["https://helm.sh/", "https://helm.sh/docs/"],
|
||||
"icon": "fas-anchor"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "liquibase",
|
||||
"name": "Liquibase",
|
||||
@@ -304,23 +261,7 @@
|
||||
"links": ["https://www.liquibase.org/", "https://docs.liquibase.com/"],
|
||||
"icon": "fas-database"
|
||||
},
|
||||
{
|
||||
"id": "ansible",
|
||||
"name": "Ansible",
|
||||
"description": "Outil d'automatisation IT sans agent pour le provisionnement, la gestion de configuration et le déploiement d'applications",
|
||||
"links": ["https://www.ansible.com/", "https://docs.ansible.com/"],
|
||||
"icon": "fas-cog"
|
||||
},
|
||||
{
|
||||
"id": "terraform",
|
||||
"name": "Terraform",
|
||||
"description": "Outil d'infrastructure as code pour la provision et la gestion des ressources cloud",
|
||||
"links": [
|
||||
"https://www.terraform.io/",
|
||||
"https://learn.hashicorp.com/terraform"
|
||||
],
|
||||
"icon": "fas-tools"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "puppet",
|
||||
"name": "Puppet",
|
||||
|
||||
@@ -110,13 +110,7 @@
|
||||
"icon": "fas-book",
|
||||
"links": ["https://storybook.js.org/", "https://storybook.js.org/docs"]
|
||||
},
|
||||
{
|
||||
"id": "cypress",
|
||||
"name": "Cypress",
|
||||
"description": "Framework de tests end-to-end",
|
||||
"icon": "fas-bug",
|
||||
"links": ["https://www.cypress.io/", "https://docs.cypress.io/"]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "playwright",
|
||||
"name": "Playwright",
|
||||
@@ -175,16 +169,7 @@
|
||||
"https://reactjs.org/docs/getting-started.html"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "typescript",
|
||||
"name": "TypeScript",
|
||||
"description": "Sur-ensemble typé de JavaScript qui se compile en JavaScript pur",
|
||||
"icon": "fab-js",
|
||||
"links": [
|
||||
"https://www.typescriptlang.org/",
|
||||
"https://www.typescriptlang.org/docs/"
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "html5-javascript-css",
|
||||
"name": "HTML5/JavaScript/CSS",
|
||||
|
||||
@@ -154,16 +154,6 @@
|
||||
"https://developer.apple.com/notifications/"
|
||||
],
|
||||
"icon": "fas-bell"
|
||||
},
|
||||
{
|
||||
"id": "react-native",
|
||||
"name": "React Native",
|
||||
"description": "Framework pour le développement d'applications mobiles natives utilisant React",
|
||||
"links": [
|
||||
"https://reactnative.dev/",
|
||||
"https://reactnative.dev/docs/getting-started"
|
||||
],
|
||||
"icon": "fab-react"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -9,7 +9,9 @@
|
||||
"start": "next start",
|
||||
"generate-test-data": "tsx scripts/generate-test-data.ts",
|
||||
"sync-skills": "tsx scripts/sync-skills.ts",
|
||||
"sync-teams": "tsx scripts/sync-teams.ts"
|
||||
"sync-teams": "tsx scripts/sync-teams.ts",
|
||||
"find-duplicates": "node scripts/find-duplicates.js",
|
||||
"clean-duplicates": "node scripts/clean-duplicates.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fortawesome/fontawesome-svg-core": "^7.0.0",
|
||||
|
||||
412
scripts/clean-duplicates.js
Normal file
412
scripts/clean-duplicates.js
Normal file
@@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
/**
|
||||
* Script pour nettoyer automatiquement les doublons dans les fichiers de skills
|
||||
* Usage: node scripts/clean-duplicates.js
|
||||
*/
|
||||
|
||||
// Configuration
|
||||
const SKILLS_DIR = path.join(__dirname, '..', 'data', 'skills');
|
||||
const BACKUP_DIR = path.join(__dirname, '..', 'backups', 'skills');
|
||||
|
||||
// Couleurs pour la console
|
||||
const colors = {
|
||||
red: '\x1b[31m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
blue: '\x1b[34m',
|
||||
magenta: '\x1b[35m',
|
||||
cyan: '\x1b[36m',
|
||||
reset: '\x1b[0m',
|
||||
bold: '\x1b[1m'
|
||||
};
|
||||
|
||||
function log(message, color = 'reset') {
|
||||
console.log(`${colors[color]}${message}${colors.reset}`);
|
||||
}
|
||||
|
||||
function logHeader(message) {
|
||||
log(`\n${colors.bold}${colors.blue}${'='.repeat(60)}${colors.reset}`);
|
||||
log(`${colors.bold}${colors.blue}${message}${colors.reset}`);
|
||||
log(`${colors.bold}${colors.blue}${'='.repeat(60)}${colors.reset}`);
|
||||
}
|
||||
|
||||
function logSubHeader(message) {
|
||||
log(`\n${colors.bold}${colors.cyan}${message}${colors.reset}`);
|
||||
log(`${colors.cyan}${'-'.repeat(message.length)}${colors.reset}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Crée une sauvegarde des fichiers avant modification
|
||||
*/
|
||||
function createBackup() {
|
||||
try {
|
||||
if (!fs.existsSync(BACKUP_DIR)) {
|
||||
fs.mkdirSync(BACKUP_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const backupPath = path.join(BACKUP_DIR, `skills-backup-${timestamp}`);
|
||||
|
||||
if (!fs.existsSync(backupPath)) {
|
||||
fs.mkdirSync(backupPath, { recursive: true });
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(SKILLS_DIR);
|
||||
files.forEach(file => {
|
||||
if (file.endsWith('.json')) {
|
||||
const sourcePath = path.join(SKILLS_DIR, file);
|
||||
const destPath = path.join(backupPath, file);
|
||||
fs.copyFileSync(sourcePath, destPath);
|
||||
}
|
||||
});
|
||||
|
||||
log(`💾 Sauvegarde créée dans: ${backupPath}`, 'green');
|
||||
return backupPath;
|
||||
} catch (error) {
|
||||
log(`❌ Erreur lors de la création de la sauvegarde: ${error.message}`, 'red');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lit et parse un fichier JSON de skills
|
||||
*/
|
||||
function parseSkillsFile(filePath) {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(content);
|
||||
} catch (error) {
|
||||
log(`❌ Erreur lors de la lecture de ${filePath}: ${error.message}`, 'red');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sauvegarde un fichier JSON de skills
|
||||
*/
|
||||
function saveSkillsFile(filePath, data) {
|
||||
try {
|
||||
const content = JSON.stringify(data, null, 2);
|
||||
fs.writeFileSync(filePath, content, 'utf8');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`❌ Erreur lors de la sauvegarde de ${filePath}: ${error.message}`, 'red');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trouve tous les fichiers de skills
|
||||
*/
|
||||
function findSkillsFiles() {
|
||||
try {
|
||||
const files = fs.readdirSync(SKILLS_DIR);
|
||||
return files
|
||||
.filter(file => file.endsWith('.json'))
|
||||
.map(file => path.join(SKILLS_DIR, file));
|
||||
} catch (error) {
|
||||
log(`❌ Erreur lors de la lecture du répertoire ${SKILLS_DIR}: ${error.message}`, 'red');
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyse tous les fichiers et collecte les informations
|
||||
*/
|
||||
function analyzeSkills() {
|
||||
const files = findSkillsFiles();
|
||||
const allSkills = [];
|
||||
const categoryStats = {};
|
||||
|
||||
files.forEach(filePath => {
|
||||
const fileName = path.basename(filePath, '.json');
|
||||
const data = parseSkillsFile(filePath);
|
||||
|
||||
if (data) {
|
||||
const category = data.category;
|
||||
const skillsCount = data.skills.length;
|
||||
|
||||
categoryStats[category] = {
|
||||
filePath,
|
||||
fileName,
|
||||
count: skillsCount,
|
||||
data
|
||||
};
|
||||
|
||||
data.skills.forEach(skill => {
|
||||
allSkills.push({
|
||||
id: skill.id,
|
||||
name: skill.name,
|
||||
category: category,
|
||||
filePath,
|
||||
fileName,
|
||||
skill
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return { allSkills, categoryStats };
|
||||
}
|
||||
|
||||
/**
|
||||
* Détecte les doublons par ID
|
||||
*/
|
||||
function findDuplicateIds(allSkills) {
|
||||
const idCounts = {};
|
||||
const duplicates = [];
|
||||
|
||||
allSkills.forEach(skill => {
|
||||
if (!idCounts[skill.id]) {
|
||||
idCounts[skill.id] = [];
|
||||
}
|
||||
idCounts[skill.id].push(skill);
|
||||
});
|
||||
|
||||
Object.entries(idCounts).forEach(([id, skills]) => {
|
||||
if (skills.length > 1) {
|
||||
duplicates.push({
|
||||
id,
|
||||
count: skills.length,
|
||||
occurrences: skills
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Détermine la catégorie la plus appropriée pour un skill
|
||||
*/
|
||||
function determineBestCategory(occurrences) {
|
||||
// Priorité des catégories (plus spécifique en premier)
|
||||
const categoryPriority = {
|
||||
'Testing': 1,
|
||||
'Security': 2,
|
||||
'Mobile': 3,
|
||||
'Design': 4,
|
||||
'Data': 5,
|
||||
'Frontend': 6,
|
||||
'Backend': 7,
|
||||
'DevOps': 8,
|
||||
'Culture': 9
|
||||
};
|
||||
|
||||
let bestOccurrence = occurrences[0];
|
||||
let bestPriority = categoryPriority[bestOccurrence.category] || 999;
|
||||
|
||||
occurrences.forEach(occurrence => {
|
||||
const priority = categoryPriority[occurrence.category] || 999;
|
||||
if (priority < bestPriority) {
|
||||
bestPriority = priority;
|
||||
bestOccurrence = occurrence;
|
||||
}
|
||||
});
|
||||
|
||||
return bestOccurrence;
|
||||
}
|
||||
|
||||
/**
|
||||
* Nettoie les doublons en gardant une version par catégorie
|
||||
*/
|
||||
function cleanDuplicates(duplicateIds, categoryStats) {
|
||||
const cleanedFiles = new Set();
|
||||
const removedSkills = [];
|
||||
|
||||
duplicateIds.forEach(duplicate => {
|
||||
const { id, occurrences } = duplicate;
|
||||
|
||||
// Détermine la meilleure catégorie
|
||||
const bestOccurrence = determineBestCategory(occurrences);
|
||||
|
||||
log(`🔧 Traitement du doublon "${id}" (${occurrences.length} occurrences)`, 'yellow');
|
||||
log(` ✅ Gardé dans: ${bestOccurrence.category} (${bestOccurrence.fileName}.json)`, 'green');
|
||||
|
||||
// Supprime les doublons des autres catégories
|
||||
occurrences.forEach(occurrence => {
|
||||
if (occurrence.filePath !== bestOccurrence.filePath) {
|
||||
const categoryData = categoryStats[occurrence.category];
|
||||
if (categoryData) {
|
||||
// Supprime le skill dupliqué
|
||||
const skillIndex = categoryData.data.skills.findIndex(s => s.id === id);
|
||||
if (skillIndex !== -1) {
|
||||
const removedSkill = categoryData.data.skills.splice(skillIndex, 1)[0];
|
||||
removedSkills.push({
|
||||
id: removedSkill.id,
|
||||
name: removedSkill.name,
|
||||
from: occurrence.category,
|
||||
file: occurrence.fileName
|
||||
});
|
||||
|
||||
log(` 🗑️ Supprimé de: ${occurrence.category} (${occurrence.fileName}.json)`, 'red');
|
||||
cleanedFiles.add(occurrence.filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return { cleanedFiles, removedSkills };
|
||||
}
|
||||
|
||||
/**
|
||||
* Sauvegarde les fichiers modifiés
|
||||
*/
|
||||
function saveModifiedFiles(cleanedFiles, categoryStats) {
|
||||
let savedCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
cleanedFiles.forEach(filePath => {
|
||||
const fileName = path.basename(filePath, '.json');
|
||||
const categoryData = Object.values(categoryStats).find(stat => stat.fileName === fileName);
|
||||
|
||||
if (categoryData && saveSkillsFile(filePath, categoryData.data)) {
|
||||
savedCount++;
|
||||
log(`💾 Fichier sauvegardé: ${fileName}.json`, 'green');
|
||||
} else {
|
||||
errorCount++;
|
||||
log(`❌ Erreur lors de la sauvegarde: ${fileName}.json`, 'red');
|
||||
}
|
||||
});
|
||||
|
||||
return { savedCount, errorCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* Génère un rapport de nettoyage
|
||||
*/
|
||||
function generateCleaningReport(duplicateIds, removedSkills, savedCount, errorCount) {
|
||||
const report = [];
|
||||
|
||||
report.push('='.repeat(80));
|
||||
report.push('RAPPORT DE NETTOYAGE DES DOUBLONS - SKILLS');
|
||||
report.push('='.repeat(80));
|
||||
report.push(`Généré le: ${new Date().toLocaleString('fr-FR')}`);
|
||||
report.push('');
|
||||
|
||||
// Résumé des actions
|
||||
report.push('📊 RÉSUMÉ DES ACTIONS');
|
||||
report.push('-'.repeat(30));
|
||||
report.push(`Doublons traités: ${duplicateIds.length}`);
|
||||
report.push(`Skills supprimés: ${removedSkills.length}`);
|
||||
report.push(`Fichiers modifiés: ${savedCount}`);
|
||||
report.push(`Erreurs: ${errorCount}`);
|
||||
report.push('');
|
||||
|
||||
// Détails des doublons traités
|
||||
if (duplicateIds.length > 0) {
|
||||
report.push('🔧 DOUBLONS TRAITÉS');
|
||||
report.push('-'.repeat(30));
|
||||
duplicateIds.forEach(duplicate => {
|
||||
const bestOccurrence = determineBestCategory(duplicate.occurrences);
|
||||
report.push(`ID: "${duplicate.id}"`);
|
||||
report.push(` ✅ Gardé dans: ${bestOccurrence.category}`);
|
||||
duplicate.occurrences.forEach(occurrence => {
|
||||
if (occurrence.filePath !== bestOccurrence.filePath) {
|
||||
report.push(` 🗑️ Supprimé de: ${occurrence.category}`);
|
||||
}
|
||||
});
|
||||
report.push('');
|
||||
});
|
||||
}
|
||||
|
||||
// Détails des skills supprimés
|
||||
if (removedSkills.length > 0) {
|
||||
report.push('🗑️ SKILLS SUPPRIMÉS');
|
||||
report.push('-'.repeat(30));
|
||||
removedSkills.forEach(skill => {
|
||||
report.push(`• ${skill.name} (ID: ${skill.id}) - Supprimé de ${skill.from} (${skill.file}.json)`);
|
||||
});
|
||||
report.push('');
|
||||
}
|
||||
|
||||
return report.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Fonction principale
|
||||
*/
|
||||
function main() {
|
||||
try {
|
||||
logHeader('NETTOYAGE AUTOMATIQUE DES DOUBLONS');
|
||||
|
||||
// Création de la sauvegarde
|
||||
logSubHeader('CRÉATION DE LA SAUVEGARDE');
|
||||
const backupPath = createBackup();
|
||||
if (!backupPath) {
|
||||
log('❌ Impossible de continuer sans sauvegarde', 'red');
|
||||
return;
|
||||
}
|
||||
|
||||
// Analyse des fichiers
|
||||
logSubHeader('ANALYSE DES FICHIERS');
|
||||
const { allSkills, categoryStats } = analyzeSkills();
|
||||
|
||||
if (allSkills.length === 0) {
|
||||
log('❌ Aucun skill trouvé. Vérifiez le répertoire.', 'red');
|
||||
return;
|
||||
}
|
||||
|
||||
log(`📁 Fichiers analysés: ${Object.keys(categoryStats).length}`);
|
||||
log(`🔍 Total de skills: ${allSkills.length}`);
|
||||
|
||||
// Détection des doublons
|
||||
logSubHeader('DÉTECTION DES DOUBLONS');
|
||||
const duplicateIds = findDuplicateIds(allSkills);
|
||||
|
||||
if (duplicateIds.length === 0) {
|
||||
log('🎉 Aucun doublon détecté ! Aucune action requise.', 'green');
|
||||
return;
|
||||
}
|
||||
|
||||
log(`🚨 Doublons détectés: ${duplicateIds.length}`, 'red');
|
||||
|
||||
// Nettoyage des doublons
|
||||
logSubHeader('NETTOYAGE DES DOUBLONS');
|
||||
const { cleanedFiles, removedSkills } = cleanDuplicates(duplicateIds, categoryStats);
|
||||
|
||||
// Sauvegarde des fichiers modifiés
|
||||
logSubHeader('SAUVEGARDE DES FICHIERS MODIFIÉS');
|
||||
const { savedCount, errorCount } = saveModifiedFiles(cleanedFiles, categoryStats);
|
||||
|
||||
// Génération du rapport
|
||||
const report = generateCleaningReport(duplicateIds, removedSkills, savedCount, errorCount);
|
||||
|
||||
// Sauvegarde du rapport
|
||||
const reportPath = path.join(__dirname, '..', 'cleaning-report.txt');
|
||||
fs.writeFileSync(reportPath, report, 'utf8');
|
||||
|
||||
// Résumé final
|
||||
logHeader('RÉSUMÉ FINAL');
|
||||
if (errorCount === 0) {
|
||||
log('🎉 Nettoyage terminé avec succès !', 'green');
|
||||
log(`📊 ${duplicateIds.length} doublons traités, ${removedSkills.length} skills supprimés`, 'green');
|
||||
log(`💾 Rapport sauvegardé dans: ${reportPath}`, 'green');
|
||||
} else {
|
||||
log(`⚠️ Nettoyage terminé avec ${errorCount} erreur(s)`, 'yellow');
|
||||
log('Consultez le rapport pour plus de détails.', 'yellow');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ Erreur fatale: ${error.message}`, 'red');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Exécution du script
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createBackup,
|
||||
analyzeSkills,
|
||||
findDuplicateIds,
|
||||
cleanDuplicates,
|
||||
generateCleaningReport
|
||||
};
|
||||
326
scripts/find-duplicates.js
Normal file
326
scripts/find-duplicates.js
Normal file
@@ -0,0 +1,326 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
/**
|
||||
* Script pour détecter les doublons dans les fichiers de skills
|
||||
* Usage: node scripts/find-duplicates.js
|
||||
*/
|
||||
|
||||
// Configuration
|
||||
const SKILLS_DIR = path.join(__dirname, '..', 'data', 'skills');
|
||||
const OUTPUT_FILE = path.join(__dirname, '..', 'duplicates-report.txt');
|
||||
|
||||
// Couleurs pour la console
|
||||
const colors = {
|
||||
red: '\x1b[31m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
blue: '\x1b[34m',
|
||||
magenta: '\x1b[35m',
|
||||
cyan: '\x1b[36m',
|
||||
reset: '\x1b[0m',
|
||||
bold: '\x1b[1m'
|
||||
};
|
||||
|
||||
function log(message, color = 'reset') {
|
||||
console.log(`${colors[color]}${message}${colors.reset}`);
|
||||
}
|
||||
|
||||
function logHeader(message) {
|
||||
log(`\n${colors.bold}${colors.blue}${'='.repeat(60)}${colors.reset}`);
|
||||
log(`${colors.bold}${colors.blue}${message}${colors.reset}`);
|
||||
log(`${colors.bold}${colors.blue}${'='.repeat(60)}${colors.reset}`);
|
||||
}
|
||||
|
||||
function logSubHeader(message) {
|
||||
log(`\n${colors.bold}${colors.cyan}${message}${colors.reset}`);
|
||||
log(`${colors.cyan}${'-'.repeat(message.length)}${colors.reset}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lit et parse un fichier JSON de skills
|
||||
*/
|
||||
function parseSkillsFile(filePath) {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
const data = JSON.parse(content);
|
||||
return {
|
||||
category: data.category,
|
||||
skills: data.skills || []
|
||||
};
|
||||
} catch (error) {
|
||||
log(`❌ Erreur lors de la lecture de ${filePath}: ${error.message}`, 'red');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trouve tous les fichiers de skills
|
||||
*/
|
||||
function findSkillsFiles() {
|
||||
try {
|
||||
const files = fs.readdirSync(SKILLS_DIR);
|
||||
return files
|
||||
.filter(file => file.endsWith('.json'))
|
||||
.map(file => path.join(SKILLS_DIR, file));
|
||||
} catch (error) {
|
||||
log(`❌ Erreur lors de la lecture du répertoire ${SKILLS_DIR}: ${error.message}`, 'red');
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyse tous les fichiers et collecte les informations
|
||||
*/
|
||||
function analyzeSkills() {
|
||||
const files = findSkillsFiles();
|
||||
const allSkills = [];
|
||||
const categoryStats = {};
|
||||
|
||||
logHeader('ANALYSE DES FICHIERS DE SKILLS');
|
||||
log(`📁 Répertoire: ${SKILLS_DIR}`);
|
||||
log(`🔍 Fichiers trouvés: ${files.length}`);
|
||||
|
||||
files.forEach(filePath => {
|
||||
const fileName = path.basename(filePath, '.json');
|
||||
const data = parseSkillsFile(filePath);
|
||||
|
||||
if (data) {
|
||||
const category = data.category;
|
||||
const skillsCount = data.skills.length;
|
||||
|
||||
// Statistiques par catégorie
|
||||
categoryStats[category] = {
|
||||
file: fileName,
|
||||
count: skillsCount,
|
||||
skills: data.skills
|
||||
};
|
||||
|
||||
// Collecte de tous les skills avec métadonnées
|
||||
data.skills.forEach(skill => {
|
||||
allSkills.push({
|
||||
id: skill.id,
|
||||
name: skill.name,
|
||||
category: category,
|
||||
file: fileName,
|
||||
description: skill.description?.substring(0, 100) + '...' || 'Pas de description'
|
||||
});
|
||||
});
|
||||
|
||||
log(`✅ ${category}: ${skillsCount} skills (${fileName}.json)`, 'green');
|
||||
}
|
||||
});
|
||||
|
||||
return { allSkills, categoryStats };
|
||||
}
|
||||
|
||||
/**
|
||||
* Détecte les doublons par ID
|
||||
*/
|
||||
function findDuplicateIds(allSkills) {
|
||||
const idCounts = {};
|
||||
const duplicates = [];
|
||||
|
||||
allSkills.forEach(skill => {
|
||||
if (!idCounts[skill.id]) {
|
||||
idCounts[skill.id] = [];
|
||||
}
|
||||
idCounts[skill.id].push(skill);
|
||||
});
|
||||
|
||||
Object.entries(idCounts).forEach(([id, skills]) => {
|
||||
if (skills.length > 1) {
|
||||
duplicates.push({
|
||||
id,
|
||||
count: skills.length,
|
||||
occurrences: skills
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Détecte les doublons par nom (avec tolérance pour les variations)
|
||||
*/
|
||||
function findDuplicateNames(allSkills) {
|
||||
const nameCounts = {};
|
||||
const duplicates = [];
|
||||
|
||||
allSkills.forEach(skill => {
|
||||
const normalizedName = skill.name.toLowerCase().trim();
|
||||
if (!nameCounts[normalizedName]) {
|
||||
nameCounts[normalizedName] = [];
|
||||
}
|
||||
nameCounts[normalizedName].push(skill);
|
||||
});
|
||||
|
||||
Object.entries(nameCounts).forEach(([name, skills]) => {
|
||||
if (skills.length > 1) {
|
||||
duplicates.push({
|
||||
name,
|
||||
count: skills.length,
|
||||
occurrences: skills
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Génère le rapport
|
||||
*/
|
||||
function generateReport(allSkills, categoryStats, duplicateIds, duplicateNames) {
|
||||
const report = [];
|
||||
|
||||
report.push('='.repeat(80));
|
||||
report.push('RAPPORT DE DÉTECTION DE DOUBLONS - SKILLS');
|
||||
report.push('='.repeat(80));
|
||||
report.push(`Généré le: ${new Date().toLocaleString('fr-FR')}`);
|
||||
report.push('');
|
||||
|
||||
// Statistiques générales
|
||||
report.push('📊 STATISTIQUES GÉNÉRALES');
|
||||
report.push('-'.repeat(30));
|
||||
report.push(`Total de skills: ${allSkills.length}`);
|
||||
report.push(`Total de catégories: ${Object.keys(categoryStats).length}`);
|
||||
report.push('');
|
||||
|
||||
// Statistiques par catégorie
|
||||
report.push('📁 STATISTIQUES PAR CATÉGORIE');
|
||||
report.push('-'.repeat(30));
|
||||
Object.entries(categoryStats).forEach(([category, stats]) => {
|
||||
report.push(`${category}: ${stats.count} skills (${stats.file}.json)`);
|
||||
});
|
||||
report.push('');
|
||||
|
||||
// Doublons par ID
|
||||
if (duplicateIds.length > 0) {
|
||||
report.push('🚨 DOUBLONS PAR ID');
|
||||
report.push('-'.repeat(30));
|
||||
duplicateIds.forEach(duplicate => {
|
||||
report.push(`ID: "${duplicate.id}" (${duplicate.count} occurrences)`);
|
||||
duplicate.occurrences.forEach(occurrence => {
|
||||
report.push(` - ${occurrence.category} (${occurrence.file}.json): ${occurrence.name}`);
|
||||
});
|
||||
report.push('');
|
||||
});
|
||||
} else {
|
||||
report.push('✅ AUCUN DOUBLON PAR ID DÉTECTÉ');
|
||||
report.push('');
|
||||
}
|
||||
|
||||
// Doublons par nom
|
||||
if (duplicateNames.length > 0) {
|
||||
report.push('⚠️ DOUBLONS PAR NOM (POTENTIELS)');
|
||||
report.push('-'.repeat(40));
|
||||
duplicateNames.forEach(duplicate => {
|
||||
report.push(`Nom: "${duplicate.name}" (${duplicate.count} occurrences)`);
|
||||
duplicate.occurrences.forEach(occurrence => {
|
||||
report.push(` - ${occurrence.category} (${occurrence.file}.json): ID="${occurrence.id}"`);
|
||||
});
|
||||
report.push('');
|
||||
});
|
||||
} else {
|
||||
report.push('✅ AUCUN DOUBLON PAR NOM DÉTECTÉ');
|
||||
report.push('');
|
||||
}
|
||||
|
||||
// Recommandations
|
||||
report.push('💡 RECOMMANDATIONS');
|
||||
report.push('-'.repeat(20));
|
||||
if (duplicateIds.length > 0) {
|
||||
report.push('• Supprimer les doublons par ID (priorité haute)');
|
||||
report.push('• Garder la version dans la catégorie la plus appropriée');
|
||||
}
|
||||
if (duplicateNames.length > 0) {
|
||||
report.push('• Vérifier les doublons par nom (peuvent être légitimes)');
|
||||
report.push('• S\'assurer que les IDs sont uniques');
|
||||
}
|
||||
if (duplicateIds.length === 0 && duplicateNames.length === 0) {
|
||||
report.push('• Aucune action requise - tous les skills sont uniques !');
|
||||
}
|
||||
|
||||
return report.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Affiche le rapport dans la console
|
||||
*/
|
||||
function displayReport(report) {
|
||||
logHeader('RAPPORT COMPLET');
|
||||
console.log(report);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sauvegarde le rapport dans un fichier
|
||||
*/
|
||||
function saveReport(report) {
|
||||
try {
|
||||
fs.writeFileSync(OUTPUT_FILE, report, 'utf8');
|
||||
log(`\n💾 Rapport sauvegardé dans: ${OUTPUT_FILE}`, 'green');
|
||||
} catch (error) {
|
||||
log(`❌ Erreur lors de la sauvegarde du rapport: ${error.message}`, 'red');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fonction principale
|
||||
*/
|
||||
function main() {
|
||||
try {
|
||||
logHeader('DÉTECTION DE DOUBLONS DANS LES SKILLS');
|
||||
|
||||
// Analyse des fichiers
|
||||
const { allSkills, categoryStats } = analyzeSkills();
|
||||
|
||||
if (allSkills.length === 0) {
|
||||
log('❌ Aucun skill trouvé. Vérifiez le répertoire.', 'red');
|
||||
return;
|
||||
}
|
||||
|
||||
// Détection des doublons
|
||||
logSubHeader('DÉTECTION DES DOUBLONS');
|
||||
const duplicateIds = findDuplicateIds(allSkills);
|
||||
const duplicateNames = findDuplicateNames(allSkills);
|
||||
|
||||
log(`🔍 Doublons par ID: ${duplicateIds.length}`, duplicateIds.length > 0 ? 'red' : 'green');
|
||||
log(`🔍 Doublons par nom: ${duplicateNames.length}`, duplicateNames.length > 0 ? 'yellow' : 'green');
|
||||
|
||||
// Génération et affichage du rapport
|
||||
const report = generateReport(allSkills, categoryStats, duplicateIds, duplicateNames);
|
||||
displayReport(report);
|
||||
|
||||
// Sauvegarde du rapport
|
||||
saveReport(report);
|
||||
|
||||
// Résumé final
|
||||
logHeader('RÉSUMÉ');
|
||||
if (duplicateIds.length === 0 && duplicateNames.length === 0) {
|
||||
log('🎉 Aucun doublon détecté ! Tous les skills sont uniques.', 'green');
|
||||
} else {
|
||||
log(`⚠️ ${duplicateIds.length} doublons par ID et ${duplicateNames.length} doublons par nom détectés.`, 'yellow');
|
||||
log('Consultez le rapport ci-dessus pour plus de détails.', 'yellow');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ Erreur fatale: ${error.message}`, 'red');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Exécution du script
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
analyzeSkills,
|
||||
findDuplicateIds,
|
||||
findDuplicateNames,
|
||||
generateReport
|
||||
};
|
||||
Reference in New Issue
Block a user