Files
espocrm/js/diff.js
2026-01-12 11:02:47 +02:00

997 lines
30 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/************************************************************************
* This file is part of EspoCRM.
*
* EspoCRM Open Source CRM application.
* Copyright (C) 2014-2026 EspoCRM, Inc.
* Website: https://www.espocrm.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU Affero General Public License version 3.
*
* In accordance with Section 7(b) of the GNU Affero General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "EspoCRM" word.
************************************************************************/
const fs = require('fs');
const cp = require('child_process');
const archiver = require('archiver');
const process = require('process');
const buildUtils = require('./build-utils');
const bundleConfig = require('../frontend/bundle-config.json');
const exec = cp.exec;
/**
* Builds upgrade packages.
*/
class Diff
{
constructor(espoPath, params) {
this.espoPath = espoPath;
this.params = params || {};
}
_getTagList() {
const dirInitial = process.cwd();
process.chdir(this.espoPath);
const tagsString = cp.execSync('git tag -l --sort=-v:refname').toString();
const tagList = tagsString.trim().split("\n");
process.chdir(dirInitial);
return tagList;
}
buildClosestUpgradePackages() {
const versionFromList = this._getPreviousVersionList(true);
this.buildMultipleUpgradePackages(versionFromList);
}
buildAllUpgradePackages() {
const versionFromList = this._getPreviousVersionList();
this.buildMultipleUpgradePackages(versionFromList);
}
_getPreviousVersionList(closest) {
const dirInitial = process.cwd();
const version = (require(this.espoPath + '/package.json') || {}).version;
process.chdir(this.espoPath);
const tagList = this._getTagList();
const versionFromList = [];
const minorVersionNumber = version.split('.')[1];
const hotfixVersionNumber = version.split('.')[2];
const majorVersionNumber = version.split('.')[0];
for (let i = 0; i < tagList.length; i++) {
const tag = tagList[i];
if (tag === '') {
continue;
}
if (tag === version) {
continue;
}
if (!~tag.indexOf('beta') && !~tag.indexOf('alpha')) {
const minorVersionNumberI = tag.split('.')[1];
if (minorVersionNumberI !== minorVersionNumber) {
versionFromList.push(tag);
break;
}
}
}
if (hotfixVersionNumber !== '0') {
for (let i = 0; i < tagList.length; i++) {
const tag = tagList[i];
const patchVersionNumberI = tag.split('.')[2];
const minorVersionNumberI = tag.split('.')[1];
const majorVersionNumberI = tag.split('.')[0];
if (
closest &&
(
minorVersionNumberI !== minorVersionNumber ||
majorVersionNumberI !== majorVersionNumber ||
parseInt(patchVersionNumberI) !== parseInt(hotfixVersionNumber) - 1
)
) {
continue;
}
if (tag === version) {
continue;
}
if (!~tag.indexOf('beta') && !~tag.indexOf('alpha')) {
versionFromList.push(tag);
if (patchVersionNumberI === '0') {
break;
}
}
}
}
process.chdir(dirInitial);
return versionFromList;
}
buildMultipleUpgradePackages(versionFromList) {
const params = this.params;
const espoPath = this.espoPath;
async function buildMultiple() {
for (const versionFrom of versionFromList) {
const diff = new Diff(espoPath, params);
await diff.buildUpgradePackage(versionFrom);
}
}
buildMultiple();
}
_versionExists(version) {
return ~this._getTagList().indexOf(version);
}
buildUpgradePackage(versionFrom) {
const params = this.params;
const espoPath = this.espoPath;
if (!this._versionExists(versionFrom)) {
throw new Error('Version ' + versionFrom + ' does not exist.');
}
return new Promise(resolve => {
const acceptedVersionName = params.acceptedVersionName || versionFrom;
const isDev = params.isDev;
const withVendor = params.withVendor ?? true;
const forceScripts = params.forceScripts;
const version = (require(espoPath + '/package.json') || {}).version;
const composerData = require(espoPath + '/composer.json') || {};
const currentPath = espoPath;
const buildRelPath = 'build/EspoCRM-' + version;
const buildPath = currentPath + '/' + buildRelPath;
const diffFilePath = currentPath + '/build/diff';
const diffBeforeUpgradeFolderPath = currentPath + '/build/diffBeforeUpgrade';
const tempFolderPath = currentPath + '/build/upgradeTmp';
const folderName = 'EspoCRM-upgrade-' + acceptedVersionName + '-to-' + version;
const upgradePath = currentPath + '/build/' + folderName;
const zipPath = currentPath + '/build/' + folderName + '.zip';
let upgradeDataFolder = versionFrom + '-' + version;
const isMinorVersion =
versionFrom.split('.')[1] !== version.split('.')[1] ||
versionFrom.split('.')[0] !== version.split('.')[0];
if (isMinorVersion) {
upgradeDataFolder = version.split('.')[0] + '.' + version.split('.')[1];
}
const upgradeDataFolderPath = currentPath + '/upgrades/' + upgradeDataFolder;
const upgradeFolderExists = fs.existsSync(upgradeDataFolderPath);
let upgradeData = {};
if (upgradeFolderExists) {
upgradeData = require(upgradeDataFolderPath + '/data.json') || {};
}
const beforeUpgradeFileList = upgradeData.beforeUpgradeFiles || [];
deleteDirRecursively(diffFilePath);
deleteDirRecursively(diffBeforeUpgradeFolderPath);
deleteDirRecursively(upgradePath);
deleteDirRecursively(tempFolderPath);
if (fs.existsSync(zipPath)) {
fs.unlinkSync(zipPath);
}
process.chdir(espoPath);
this._notifyIfBadBranch();
if (!fs.existsSync(buildPath)) {
throw new Error(
"EspoCRM is not built. You need to run 'grunt' before building an upgrade package."
);
}
if (!fs.existsSync(upgradePath)) {
fs.mkdirSync(upgradePath);
}
if (!fs.existsSync(upgradePath + '/files')) {
fs.mkdirSync(upgradePath + '/files');
}
if (fs.existsSync(upgradeDataFolderPath + '/beforeUpgradeFiles')) {
cp.execSync(
'cp -r ' + upgradeDataFolderPath + '/beforeUpgradeFiles ' + upgradePath + '/beforeUpgradeFiles'
);
}
if (beforeUpgradeFileList.length) {
if (!fs.existsSync(upgradePath + '/beforeUpgradeFiles')) {
fs.mkdirSync(upgradePath + '/beforeUpgradeFiles');
}
}
const libData = this._getLibData({
versionFrom: versionFrom,
currentPath: currentPath,
});
const deleteFileList = this._getDeletedFileList(versionFrom)
.concat(libData.filesToDelete)
.filter((item, i, list) => list.indexOf(item) === i);
const tagList = this._getTagList();
process.chdir(buildPath);
let fileList = upgradeData.mandatoryFiles || [];
const stdout = cp.execSync('git diff --name-only ' + versionFrom).toString();
(stdout || '').trim().split('\n').forEach(file => {
if (file === '') {
return;
}
fileList.push(file);
});
libData.filesToCopy.forEach(item => fileList.push(item));
fileList.push('application/Espo/Resources/defaults.php');
fileList.push('client/lib/espo.js');
fileList.push('client/lib/espo.js.map');
fileList.push('client/lib/templates.tpl');
fileList.push('client/lib/original/espo.js');
Object.keys(bundleConfig.chunks)
.map(name => {
const namePart = `espo-${name}`;
fileList.push(`client/lib/${namePart}.js`);
fileList.push(`client/lib/${namePart}.js.map`);
fileList.push(`client/lib/original/${namePart}.js`);
});
fs.readdirSync('client/css/espo/').forEach(file => {
fileList.push('client/css/espo/' + file);
});
fileList = fileList.filter(item => fs.existsSync(item));
fs.writeFileSync(diffFilePath, fileList.join('\n'));
if (beforeUpgradeFileList.length) {
fs.writeFileSync(diffBeforeUpgradeFolderPath, beforeUpgradeFileList.join('\n'));
}
if (beforeUpgradeFileList.length) {
cp.execSync(
'xargs -a ' + diffBeforeUpgradeFolderPath +
' cp -p --parents -t ' + upgradePath + '/beforeUpgradeFiles'
);
}
if (!isDev || forceScripts) {
if (fs.existsSync(upgradeDataFolderPath + '/scripts')) {
cp.execSync('cp -r ' + upgradeDataFolderPath + '/scripts ' + upgradePath + '/scripts');
}
}
execute('xargs -a ' + diffFilePath + ' cp -p --parents -t ' + upgradePath + '/files' , () => {
const date = this._getCurrentDate();
let versionList = [];
tagList.forEach(tag => {
if (tag === versionFrom) {
versionList.push(tag);
}
/*if (!tag || tag === version) {
return;
}*/
});
if (isDev) {
versionList = [];
}
const upgradeName = acceptedVersionName + " to " + version;
const manifestData = {
"name": "EspoCRM Upgrade " + upgradeName,
"type": "upgrade",
"version": version,
"acceptableVersions": versionList,
"php": [composerData.require.php],
"releaseDate": date,
"author": "EspoCRM",
"description": "",
"delete": deleteFileList,
};
const additionalManifestData = upgradeData.manifest || {};
for (const item in additionalManifestData) {
if (Array.isArray(manifestData[item])) {
manifestData[item] = manifestData[item].concat(additionalManifestData[item]);
continue;
}
manifestData[item] = additionalManifestData[item];
}
fs.writeFileSync(upgradePath + '/manifest.json', JSON.stringify(manifestData, null, ' '));
if (fs.existsSync(diffFilePath)) {
fs.unlinkSync(diffFilePath);
}
if (fs.existsSync(diffBeforeUpgradeFolderPath)) {
fs.unlinkSync(diffBeforeUpgradeFolderPath);
}
if (withVendor) {
this._processVendor({
versionFrom: versionFrom,
currentPath: currentPath,
tempFolderPath: tempFolderPath,
upgradePath: upgradePath,
});
}
this._processArchive({
upgradeName: upgradeName,
zipPath: zipPath,
upgradePath: upgradePath,
})
.then(() => resolve());
});
});
}
_notifyIfBadBranch() {
const currentBranch = cp.execSync('git rev-parse --abbrev-ref HEAD').toString().trim();
if (
currentBranch !== 'master' &&
currentBranch !== 'stable' &&
currentBranch !== 'fix'
) {
console.log('\x1b[33m%s\x1b[0m', "Warning! You are on " + currentBranch + " branch.");
}
}
_getCurrentDate() {
const d = new Date();
let monthN = ((d.getMonth() + 1).toString());
monthN = monthN.length === 1 ? '0' + monthN : monthN;
let dateN = d.getDate().toString();
dateN = dateN.length === 1 ? '0' + dateN : dateN;
return d.getFullYear().toString() + '-' + monthN + '-' + dateN.toString();
}
_getDeletedFileList(versionFrom) {
const dirInitial = process.cwd();
process.chdir(this.espoPath);
let deletedFileList = this._getRepositoryDeletedFileList(versionFrom);
const previousAllFileList = this._getPreviousAllFileList(versionFrom);
const actualAllFileList = this._getActualAllFileList();
previousAllFileList.forEach(file => {
if (
!~actualAllFileList.indexOf(file) &&
!~deletedFileList.indexOf(file)
) {
deletedFileList.push(file);
}
});
deletedFileList = deletedFileList.filter(item => {
if (
item.indexOf('tests/') === 0 ||
item.indexOf('upgrades/') === 0 ||
item.indexOf('frontend/') === 0||
item.indexOf('js/') === 0
) {
return false;
}
return true;
});
process.chdir(dirInitial);
return deletedFileList;
}
_getRepositoryDeletedFileList(versionFrom) {
const deletedFileList = [];
const stdout = cp.execSync('git diff --name-only --diff-filter=D ' + versionFrom).toString();
(stdout || '').trim().split('\n').forEach(file => {
if (file === '') {
return;
}
deletedFileList.push(file);
});
return deletedFileList;
}
_getActualAllFileList() {
const actualAllFileList = [];
const stdout = cp.execSync('git ls-tree -r --name-only HEAD').toString();
(stdout || '').trim().split('\n').forEach(file => {
if (file === '') {
return;
}
actualAllFileList.push(file);
});
return actualAllFileList;
}
_getPreviousAllFileList(versionFrom) {
const previousAllFileList = [];
const stdout = cp.execSync('git ls-tree -r --name-only ' + versionFrom).toString();
(stdout || '').trim().split('\n').forEach(file => {
if (file === '') {
return;
}
previousAllFileList.push(file);
});
return previousAllFileList;
}
_deleteGitFolderInVendor(dir) {
const folderList = fs.readdirSync(dir, {withFileTypes: true})
.filter(dirent => dirent.isDirectory())
.map(dirent => dirent.name);
folderList.forEach(folder => {
const path = dir + '/' + folder;
const gitPath = path + '/.git';
if (fs.existsSync(gitPath)) {
deleteDirRecursively(gitPath);
}
});
}
_getLibData(dto) {
const data = {
filesToDelete: [],
filesToCopy: [],
};
const versionFrom = dto.versionFrom;
const currentPath = dto.currentPath;
const output = cp.execSync("git show " + versionFrom + " --format=%H").toString();
const commitHash = output.trim().split("\n")[3];
if (!commitHash) {
throw new Error("Couldn't find commit hash.");
}
const packageLockOldContents = cp.execSync("git show " + commitHash + ":package-lock.json").toString();
const packageLockNewContents = cp.execSync("cat " + currentPath + "/package-lock.json").toString();
const depsOld = JSON.parse(packageLockOldContents).dependencies || {};
const depsNew = JSON.parse(packageLockNewContents).dependencies || {};
if (packageLockOldContents === packageLockNewContents) {
return data;
}
let libOldDataList = [];
let bundledOldDataList = [];
if (~this._getVersionAllFileList(versionFrom).indexOf('frontend/libs.json')) {
libOldDataList = JSON
.parse(
cp.execSync("git show " + commitHash + ":frontend/libs.json").toString() || '[]'
)
.filter(item => !item.bundle);
bundledOldDataList = JSON
.parse(
cp.execSync("git show " + commitHash + ":frontend/libs.json").toString() || '[]'
)
.filter(item => item.bundle)
.filter(item => item.src || item.files);
}
const libNewDataList = require(this.espoPath + '/frontend/libs.json')
.filter(item => !item.bundle);
const bundledNewDataList = require(this.espoPath + '/frontend/libs.json')
.filter(item => item.bundle)
.filter(item => item.src || item.files);
const resolveItemDest = item =>
item.dest || 'client/lib/' + item.src.split('/').pop();
/*const resolveBundledItemDest = item => {
if (item.amdId) {
return `client/lib/original/${item.amdId}.js`;
}
return 'client/lib/original/' + item.src.split('/').pop();
};*/
const resolveItemName = item => {
if (item.name) {
return item.name;
}
const src = /** @type string */item.src;
if (!src) {
throw new Error("Bad lib data in `frontend/libs.json`.");
}
const name = src.split('/')[1];
if (!name) {
throw new Error("Bad lib data in `frontend/libs.json`.");
}
if (name.startsWith('@')) {
return src.split('/').slice(1, 3).join('/');
}
return name;
};
const changedLibList = [];
const currentLibList = [];
const changedBundledList = [];
const currentBundledList = [];
const toMinifyOldMap = {};
const libOldDataMap = {};
const bundledOldDataMap = {};
libOldDataList.forEach(item => {
const name = resolveItemName(item);
toMinifyOldMap[name] = item.minify || false;
libOldDataMap[name] = item;
});
bundledOldDataList.forEach(item => {
const name = resolveItemName(item);
bundledOldDataMap[name] = item;
})
libNewDataList.forEach(item => {
const name = resolveItemName(item);
const minify = item.minify || false;
if (!depsNew[name]) {
throw new Error("Not installed lib '" + name + "' `frontend/libs.json`.");
}
currentLibList.push(name);
const isAdded = !(name in depsOld);
const versionNew = depsNew[name].version || null;
const versionOld = (depsOld[name] || {}).version || null;
const wasMinified = (toMinifyOldMap || {})[name];
const isDefsChanged = libOldDataMap[name] ?
JSON.stringify(item) !== JSON.stringify(libOldDataMap[name]) :
false;
if (
!isAdded &&
versionNew === versionOld &&
minify === wasMinified &&
!isDefsChanged
) {
return;
}
changedLibList.push(name);
if (item.files) {
item.files.forEach(item =>
data.filesToCopy.push(resolveItemDest(item))
);
if (minify) {
item.files.forEach(item => {
data.filesToCopy.push(resolveItemDest(item) + '.map');
data.filesToCopy.push(
buildUtils.destToOriginalDest(resolveItemDest(item))
);
});
}
return;
}
data.filesToCopy.push(resolveItemDest(item));
if (minify) {
data.filesToCopy.push(resolveItemDest(item) + '.map');
data.filesToCopy.push(
buildUtils.destToOriginalDest(resolveItemDest(item))
);
}
});
libOldDataList.forEach(item => {
const name = resolveItemName(item);
const minify = item.minify || false;
let toRemove = false;
if (
~changedLibList.indexOf(name) ||
!~currentLibList.indexOf(name)
) {
toRemove = true;
}
if (!toRemove) {
return;
}
if (item.files) {
item.files.forEach(item =>
data.filesToDelete.push(resolveItemDest(item))
);
if (minify) {
item.files.forEach(item => {
data.filesToDelete.push(resolveItemDest(item) + '.map');
data.filesToDelete.push(
buildUtils.destToOriginalDest(resolveItemDest(item))
);
});
}
return;
}
data.filesToDelete.push(resolveItemDest(item));
if (minify) {
data.filesToDelete.push(resolveItemDest(item) + '.map');
data.filesToDelete.push(
buildUtils.destToOriginalDest(resolveItemDest(item))
);
}
});
bundledNewDataList.forEach(item => {
const name = resolveItemName(item);
if (!depsNew[name]) {
throw new Error("Not installed lib '" + name + "' `frontend/libs.json`.");
}
currentBundledList.push(name);
const isAdded = !(name in depsOld);
const versionNew = depsNew[name].version || null;
const versionOld = (depsOld[name] || {}).version || null;
const isDefsChanged = libOldDataMap[name] ?
JSON.stringify(item) !== JSON.stringify(libOldDataMap[name]) :
false;
if (
!isAdded &&
versionNew === versionOld &&
!isDefsChanged
) {
return;
}
changedBundledList.push(name);
/*if (item.files) {
item.files.forEach(item =>
data.filesToCopy.push(resolveBundledItemDest(item))
);
return;
}
if (!item.src) {
return;
}
data.filesToCopy.push(resolveBundledItemDest(item));*/
});
bundledOldDataList.forEach(item => {
const name = resolveItemName(item);
let toRemove = false;
if (
~changedBundledList.indexOf(name) ||
!~currentBundledList.indexOf(name)
) {
toRemove = true;
}
if (!toRemove) {
// noinspection UnnecessaryReturnStatementJS
return;
}
/*if (item.files) {
item.files.forEach(item =>
data.filesToDelete.push(resolveBundledItemDest(item))
);
return;
}
if (!item.src) {
return;
}
data.filesToDelete.push(resolveBundledItemDest(item));*/
});
return data;
}
_getVersionAllFileList(version) {
const output = cp.execSync("git show " + version + " --format=%H").toString();
const commitHash = output.trim().split("\n")[3];
const list = [];
cp.execSync("git ls-tree -r " + commitHash + " --name-only")
.toString()
.trim()
.split('\n')
.forEach(file => {
if (file === '') {
return;
}
list.push(file);
});
return list;
}
_processVendor(dto) {
const versionFrom = dto.versionFrom;
const currentPath = dto.currentPath;
const tempFolderPath = dto.tempFolderPath;
const upgradePath = dto.upgradePath;
const output = cp.execSync("git show " + versionFrom + " --format=%H").toString();
const commitHash = output.trim().split("\n")[3];
if (!commitHash) {
throw new Error("Couldn't find commit hash.");
}
const composerLockOldContents = cp.execSync("git show " + commitHash + ":composer.lock").toString();
const composerOldContents = cp.execSync("git show " + commitHash + ":composer.json").toString();
let composerLockNewContents = cp.execSync("cat " + currentPath + "/composer.lock").toString();
let composerNewContents = cp.execSync("cat " + currentPath + "/composer.json").toString();
composerNewContents = composerNewContents.replace(/(\r\n)/g, '\n');
composerLockNewContents = composerLockNewContents.replace(/(\r\n)/g, '\n');
if (
composerLockNewContents === composerLockOldContents &&
composerOldContents === composerNewContents
) {
return;
}
const newPackages = JSON.parse(composerLockNewContents).packages;
const oldPackages = JSON.parse(composerLockOldContents).packages;
fs.mkdirSync(tempFolderPath);
fs.mkdirSync(tempFolderPath + '/new');
const vendorPath = tempFolderPath + '/new/vendor/';
cp.execSync(`cp -r ${currentPath}/dev ${tempFolderPath}/new/dev`);
fs.writeFileSync(tempFolderPath + '/new/composer.lock', composerLockNewContents);
fs.writeFileSync(tempFolderPath + '/new/composer.json', composerNewContents);
cp.execSync(
"composer install --no-dev --ignore-platform-reqs",
{
cwd: tempFolderPath + "/new",
stdio: 'ignore',
}
);
fs.mkdirSync(upgradePath + '/vendorFiles');
cp.execSync("mv " + vendorPath + "/autoload.php " + upgradePath + "/vendorFiles/autoload.php");
cp.execSync("mv " + vendorPath + "/composer " + upgradePath + "/vendorFiles/composer");
cp.execSync("mv " + vendorPath + "/bin "+ upgradePath + "/vendorFiles/bin");
const folderList = [];
for (const item of newPackages) {
const name = item.name;
if (name.indexOf('composer/') === 0) {
continue;
}
let isFound = false;
let toAdd = false;
for (const oItem of oldPackages) {
if (oItem.name !== name) {
continue;
}
isFound = true;
if (item.version !== oItem.version) {
toAdd = true;
}
}
if (!isFound) {
toAdd = true;
}
if (toAdd) {
const folder = name.split('/')[0];
if (!~folderList.indexOf(folder)) {
folderList.push(folder);
}
}
}
for (const folder of folderList) {
this._deleteGitFolderInVendor(vendorPath + '/' + folder);
if (fs.existsSync(vendorPath + '/'+ folder)) {
cp.execSync(
'mv ' + vendorPath + '/' + folder + " " + upgradePath + '/vendorFiles/' + folder
);
}
}
deleteDirRecursively(tempFolderPath);
}
_processArchive(dto) {
const zipPath = dto.zipPath;
const upgradePath = dto.upgradePath;
const upgradeName = dto.upgradeName;
return new Promise(resolve => {
const zipOutput = fs.createWriteStream(zipPath);
const archive = archiver('zip');
archive.on('error', err => {
throw err;
});
zipOutput.on('close', () => {
console.log("Upgrade package has been built: " + upgradeName + "");
deleteDirRecursively(upgradePath);
resolve();
});
archive.directory(upgradePath, false).pipe(zipOutput);
archive.finalize();
});
}
}
function deleteDirRecursively(path) {
if (fs.existsSync(path) && fs.lstatSync(path).isDirectory()) {
fs.readdirSync(path).forEach(file => {
const curPath = path + "/" + file;
if (fs.lstatSync(curPath).isDirectory()) {
deleteDirRecursively(curPath);
}
else {
fs.unlinkSync(curPath);
}
});
fs.rmdirSync(path);
return;
}
if (fs.existsSync(path) && fs.lstatSync(path).isFile()) {
fs.unlinkSync(path);
}
}
function execute(command, callback) {
exec(command, (error, stdout) => callback(stdout));
}
module.exports = Diff;