From 2399d21829b0771ba48a25eea7b80232904260eb Mon Sep 17 00:00:00 2001 From: Yuri Kuznetsov Date: Tue, 28 Feb 2023 12:12:21 +0200 Subject: [PATCH] PostgreSQL (#2599) * dev * postgres update set * position in list * discard charset * binary => blob * rollback transaction * insert on conflict * fix POSITION_IN_LIST * TIMESTAMPDIFF * functions * functions * set UTC time zone * functions and operators * function * fulltext * fix details provider * full text config usage * fix param * full text index rebuild * full text and round fix * add uuid db type * if function * tests * delete with joins order limit * update * alias max length --- .../Factories/PostgresqlConnectionFactory.php | 9 +- .../Platforms/PostgreSQLSchemaManager.php | 98 +++ .../Dbal/Platforms/PostgresqlPlatform.php | 83 +++ .../PostgresqlDetailsProvider.php | 106 ++++ .../IndexHelpers/PostgresqlIndexHelper.php | 81 +++ .../PostgresqlColumnPreparator.php | 148 +++++ .../Espo/ORM/PDO/PostgresqlPDOFactory.php | 81 +++ .../QueryComposer/PostgresqlQueryComposer.php | 577 ++++++++++++++++++ .../metadata/app/databasePlatforms.json | 8 +- .../Espo/Resources/metadata/app/orm.json | 9 +- .../Espo/ORM/PostgresqlQueryComposerTest.php | 209 +++++++ tests/unit/testData/DB/ormMetadata.php | 9 +- 12 files changed, 1413 insertions(+), 5 deletions(-) create mode 100644 application/Espo/Core/Utils/Database/Dbal/Platforms/PostgreSQLSchemaManager.php create mode 100644 application/Espo/Core/Utils/Database/Dbal/Platforms/PostgresqlPlatform.php create mode 100644 application/Espo/Core/Utils/Database/DetailsProviders/PostgresqlDetailsProvider.php create mode 100644 application/Espo/Core/Utils/Database/Orm/IndexHelpers/PostgresqlIndexHelper.php create mode 100644 application/Espo/Core/Utils/Database/Schema/ColumnPreparators/PostgresqlColumnPreparator.php create mode 100644 application/Espo/ORM/PDO/PostgresqlPDOFactory.php create mode 100644 application/Espo/ORM/QueryComposer/PostgresqlQueryComposer.php create mode 100644 tests/unit/Espo/ORM/PostgresqlQueryComposerTest.php diff --git a/application/Espo/Core/Utils/Database/Dbal/Factories/PostgresqlConnectionFactory.php b/application/Espo/Core/Utils/Database/Dbal/Factories/PostgresqlConnectionFactory.php index 3a98e8d87f..b39793f557 100644 --- a/application/Espo/Core/Utils/Database/Dbal/Factories/PostgresqlConnectionFactory.php +++ b/application/Espo/Core/Utils/Database/Dbal/Factories/PostgresqlConnectionFactory.php @@ -33,6 +33,8 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\Driver\PDO\PgSQL\Driver as PostgreSQLDriver; use Doctrine\DBAL\Exception as DBALException; use Espo\Core\Utils\Database\Dbal\ConnectionFactory; +use Espo\Core\Utils\Database\Dbal\Platforms\PostgresqlPlatform; +use Espo\Core\Utils\Database\Helper; use Espo\ORM\DatabaseParams; use Espo\ORM\PDO\Options as PdoOptions; @@ -42,7 +44,8 @@ use RuntimeException; class PostgresqlConnectionFactory implements ConnectionFactory { public function __construct( - private PDO $pdo + private PDO $pdo, + private Helper $helper ) {} /** @@ -56,7 +59,11 @@ class PostgresqlConnectionFactory implements ConnectionFactory throw new RuntimeException("No required database params."); } + $platform = new PostgresqlPlatform(); + $platform->setTextSearchConfig($this->helper->getParam('default_text_search_config')); + $params = [ + 'platform' => $platform, 'pdo' => $this->pdo, 'host' => $databaseParams->getHost(), 'dbname' => $databaseParams->getName(), diff --git a/application/Espo/Core/Utils/Database/Dbal/Platforms/PostgreSQLSchemaManager.php b/application/Espo/Core/Utils/Database/Dbal/Platforms/PostgreSQLSchemaManager.php new file mode 100644 index 0000000000..2972a88e59 --- /dev/null +++ b/application/Espo/Core/Utils/Database/Dbal/Platforms/PostgreSQLSchemaManager.php @@ -0,0 +1,98 @@ +_conn->fetchAllAssociative($sql); + + if (!$rows) { + continue; + } + + $columns = self::parseColumnsIndexFromDeclaration($rows[0]['indexdef']); + + $indexes[$key] = new Index( + $key, + $columns, + false, + false, + ['fulltext'] + ); + } + } + + return $indexes; + } + + /** + * @return string[] + */ + private static function parseColumnsIndexFromDeclaration(string $string): array + { + preg_match('/to_tsvector\((.*),(.*)\)/i', $string, $matches); + + if (!$matches || count($matches) < 3) { + return []; + } + + $part = $matches[2]; + + $part = str_replace("|| ' '::text", '', $part); + $part = str_replace("::text", '', $part); + $part = str_replace(" ", '', $part); + $part = str_replace("||", ' ', $part); + $part = str_replace("(", '', $part); + $part = str_replace(")", '', $part); + + $list = array_map( + fn ($item) => trim($item), + explode(' ', $part) + ); + + return $list; + } +} diff --git a/application/Espo/Core/Utils/Database/Dbal/Platforms/PostgresqlPlatform.php b/application/Espo/Core/Utils/Database/Dbal/Platforms/PostgresqlPlatform.php new file mode 100644 index 0000000000..60db3e17d3 --- /dev/null +++ b/application/Espo/Core/Utils/Database/Dbal/Platforms/PostgresqlPlatform.php @@ -0,0 +1,83 @@ +textSearchConfig = $textSearchConfig; + } + + public function createSchemaManager(Connection $connection): PostgreSQLSchemaManager + { + return new PostgreSQLSchemaManager($connection, $this); + } + + public function getCreateIndexSQL(Index $index, $table) + { + if (!$index->hasFlag('fulltext')) { + return parent::getCreateIndexSQL($index, $table); + } + + if ($table instanceof Table) { + $table = $table->getQuotedName($this); + } + + $name = $index->getQuotedName($this); + $columns = $index->getColumns(); + + if (count($columns) === 0) { + throw new \InvalidArgumentException(sprintf( + 'Incomplete or invalid index definition %s on table %s', + $name, + $table, + )); + } + + $columnsPart = implode(" || ' ' || ", $index->getQuotedColumns($this)); + $partialPart = $this->getPartialIndexSQL($index); + + $textSearchConfig = $this->textSearchConfig ?? self::TEXT_SEARCH_CONFIG; + $textSearchConfig = preg_replace('/[^A-Za-z0-9_.\-]+/', '', $textSearchConfig) ?? ''; + $configPart = $this->quoteStringLiteral($textSearchConfig); + + return "CREATE INDEX {$name} ON {$table} USING GIN (TO_TSVECTOR({$configPart}, {$columnsPart})) {$partialPart}"; + } +} diff --git a/application/Espo/Core/Utils/Database/DetailsProviders/PostgresqlDetailsProvider.php b/application/Espo/Core/Utils/Database/DetailsProviders/PostgresqlDetailsProvider.php new file mode 100644 index 0000000000..7f822e0eab --- /dev/null +++ b/application/Espo/Core/Utils/Database/DetailsProviders/PostgresqlDetailsProvider.php @@ -0,0 +1,106 @@ +getFullDatabaseVersion() ?? ''; + + if (preg_match('/[0-9]+\.[0-9]+/', $fullVersion, $match)) { + return $match[0]; + } + + return '0.0'; + } + + public function getServerVersion(): string + { + return (string) $this->getParam('version'); + } + + public function getParam(string $name): ?string + { + $name = preg_replace('/[^A-Za-z0-9_]+/', '', $name);; + + $sql = "SHOW {$name}"; + + $sth = $this->pdo->query($sql); + + if ($sth === false) { + return null; + } + + $row = $sth->fetch(PDO::FETCH_NUM); + + if ($row === false) { + return null; + } + + $value = $row[0] ?: null; + + if ($value === null) { + return null; + } + + return (string) $value; + } + + private function getFullDatabaseVersion(): ?string + { + $sql = "select version()"; + + $sth = $this->pdo->prepare($sql); + $sth->execute(); + + /** @var string|null|false $result */ + $result = $sth->fetchColumn(); + + if ($result === false || $result === null) { + return null; + } + + return $result; + } +} diff --git a/application/Espo/Core/Utils/Database/Orm/IndexHelpers/PostgresqlIndexHelper.php b/application/Espo/Core/Utils/Database/Orm/IndexHelpers/PostgresqlIndexHelper.php new file mode 100644 index 0000000000..3a3dbb37f9 --- /dev/null +++ b/application/Espo/Core/Utils/Database/Orm/IndexHelpers/PostgresqlIndexHelper.php @@ -0,0 +1,81 @@ +getName(); + $prefix = $defs->isUnique() ? 'UNIQ' : 'IDX'; + + $parts = [ + $prefix, + strtoupper(Util::toUnderScore($entityType)), + strtoupper(Util::toUnderScore($name)), + ]; + + $key = implode('_', $parts); + + return self::decreaseLength($key); + } + + private static function decreaseLength(string $key): string + { + if (strlen($key) <= self::MAX_LENGTH) { + return $key; + } + + $list = explode('_', $key); + + $maxItemLength = 0; + foreach ($list as $item) { + if (strlen($item) > $maxItemLength) { + $maxItemLength = strlen($item); + } + } + $maxItemLength--; + + $list = array_map( + fn ($item) => substr($item, 0, min($maxItemLength, strlen($item))), + $list + ); + + $key = implode('_', $list); + + return self::decreaseLength($key); + } +} diff --git a/application/Espo/Core/Utils/Database/Schema/ColumnPreparators/PostgresqlColumnPreparator.php b/application/Espo/Core/Utils/Database/Schema/ColumnPreparators/PostgresqlColumnPreparator.php new file mode 100644 index 0000000000..fff48ce372 --- /dev/null +++ b/application/Espo/Core/Utils/Database/Schema/ColumnPreparators/PostgresqlColumnPreparator.php @@ -0,0 +1,148 @@ + */ + private array $columnTypeMap = [ + Entity::BOOL => Types::BOOLEAN, + Entity::INT => Types::INTEGER, + Entity::VARCHAR => Types::STRING, + // DBAL reverse engineers as blob. + Types::BINARY => Types::BLOB, + ]; + + public function __construct() {} + + public function prepare(AttributeDefs $defs): Column + { + $dbType = $defs->getParam(self::PARAM_DB_TYPE); + $type = $defs->getType(); + $length = $defs->getLength(); + $default = $defs->getParam(self::PARAM_DEFAULT); + $notNull = $defs->getParam(self::PARAM_NOT_NULL); + $autoincrement = $defs->getParam(self::PARAM_AUTOINCREMENT); + $precision = $defs->getParam(self::PARAM_PRECISION); + $scale = $defs->getParam(self::PARAM_SCALE); + + $columnType = $dbType ?? $type; + + if (in_array($type, $this->textTypeList) && !$dbType) { + $columnType = Types::TEXT; + } + + $columnType = $this->columnTypeMap[$columnType] ?? $columnType; + + $columnName = Util::toUnderScore($defs->getName()); + + $column = Column::create($columnName, strtolower($columnType)); + + if ($length !== null) { + $column = $column->withLength($length); + } + + if ($default !== null) { + $column = $column->withDefault($default); + } + + if ($notNull !== null) { + $column = $column->withNotNull($notNull); + } + + if ($autoincrement !== null) { + $column = $column->withAutoincrement($autoincrement); + } + + if ($precision !== null) { + $column = $column->withPrecision($precision); + } + + if ($scale !== null) { + $column = $column->withScale($scale); + } + + switch ($type) { + case Entity::TEXT: + case Entity::JSON_ARRAY: + $column = $column->withDefault(null); + + break; + + case Entity::BOOL: + $default = intval($default ?? false); + + $column = $column->withDefault($default); + + break; + } + + if ($type !== Entity::ID && $autoincrement) { + $column = $column + ->withNotNull() + ->withUnsigned(); + } + + return $column; + + // @todo Revise. Comparator would detect the column as changed if charset is set. + /*if ( + !in_array($columnType, [ + Types::STRING, + Types::TEXT, + ]) + ) { + return $column; + } + + return $column->withCharset('UTF8');*/ + } +} diff --git a/application/Espo/ORM/PDO/PostgresqlPDOFactory.php b/application/Espo/ORM/PDO/PostgresqlPDOFactory.php new file mode 100644 index 0000000000..887991590a --- /dev/null +++ b/application/Espo/ORM/PDO/PostgresqlPDOFactory.php @@ -0,0 +1,81 @@ +getPlatform() ?? ''); + + $host = $databaseParams->getHost(); + $port = $databaseParams->getPort(); + $dbname = $databaseParams->getName(); + //$charset = $databaseParams->getCharset(); + $username = $databaseParams->getUsername(); + $password = $databaseParams->getPassword(); + + if (!$platform) { + throw new RuntimeException("No 'platform' parameter."); + } + + if (!$host) { + throw new RuntimeException("No 'host' parameter."); + } + + $dsn = 'pgsql:' . 'host=' . $host; + + if ($port) { + $dsn .= ';' . 'port=' . (string) $port; + } + + if ($dbname) { + $dsn .= ';' . 'dbname=' . $dbname; + } + + /*if ($charset) { + $dsn .= ';' . 'options=' . "'--client_encoding={$charset}'"; + }*/ + + $options = Options::getOptionsFromDatabaseParams($databaseParams); + + $pdo = new PDO($dsn, $username, $password, $options); + + $pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + + $pdo->query("SET time zone 'UTC'"); + + return $pdo; + } +} diff --git a/application/Espo/ORM/QueryComposer/PostgresqlQueryComposer.php b/application/Espo/ORM/QueryComposer/PostgresqlQueryComposer.php new file mode 100644 index 0000000000..cbf293be35 --- /dev/null +++ b/application/Espo/ORM/QueryComposer/PostgresqlQueryComposer.php @@ -0,0 +1,577 @@ + */ + protected array $comparisonOperators = [ + '!=s' => 'NOT IN', + '=s' => 'IN', + '!=' => '<>', + '!*' => 'NOT ILIKE', + '*' => 'ILIKE', + '>=' => '>=', + '<=' => '<=', + '>' => '>', + '<' => '<', + '=' => '=', + ]; + + /** @var array */ + protected array $comparisonFunctionOperatorMap = [ + 'LIKE' => 'ILIKE', + 'NOT_LIKE' => 'NOT ILIKE', + 'EQUAL' => '=', + 'NOT_EQUAL' => '<>', + 'GREATER_THAN' => '>', + 'LESS_THAN' => '<', + 'GREATER_THAN_OR_EQUAL' => '>=', + 'LESS_THAN_OR_EQUAL' => '<=', + 'IS_NULL' => 'IS NULL', + 'IS_NOT_NULL' => 'IS NOT NULL', + 'IN' => 'IN', + 'NOT_IN' => 'NOT IN', + ]; + + protected function quoteColumn(string $column): string + { + $list = explode('.', $column); + $list = array_map(fn ($item) => '"' . $item . '"', $list); + + return implode('.', $list); + } + + /** + * @todo Make protected. + * + * @param mixed $value + */ + public function quote($value): string + { + if (is_null($value)) { + return 'NULL'; + } + + if (is_bool($value)) { + return $value ? 'true' : 'false'; + } + + if (is_int($value)) { + return strval($value); + } + + if (is_float($value)) { + return strval($value); + } + + return $this->pdo->quote($value); + } + + /** + * @param string[] $argumentPartList + * @param array $params + */ + protected function getFunctionPart( + string $function, + string $part, + array $params, + string $entityType, + bool $distinct, + array $argumentPartList = [] + ): string { + + if (in_array($function, ['MATCH_BOOLEAN', 'MATCH_NATURAL_LANGUAGE'])) { + if (count($argumentPartList) < 2) { + throw new RuntimeException("Not enough arguments for MATCH function."); + } + + $queryPart = end($argumentPartList); + $columnsPart = implode( + " || ' ' || ", + array_map( + fn ($item) => "COALESCE({$item}, '')", + array_slice($argumentPartList, 0, -1) + ) + ); + + return "TS_RANK_CD(TO_TSVECTOR({$columnsPart}), PLAINTO_TSQUERY({$queryPart}))"; + } + + if ($function === 'IF') { + if (count($argumentPartList) < 3) { + throw new RuntimeException("Not enough arguments for IF function."); + } + + $conditionPart = $argumentPartList[0]; + $thenPart = $argumentPartList[1]; + $elsePart = $argumentPartList[2]; + + return "CASE WHEN {$conditionPart} THEN {$thenPart} ELSE {$elsePart} END"; + } + + if ($function === 'ROUND') { + if (count($argumentPartList) === 2 && $argumentPartList[1] === '0') { + $argumentPartList = array_slice($argumentPartList, 0, -1); + + return "ROUND({$argumentPartList[0]})"; + } + } + + if ($function === 'UNIX_TIMESTAMP') { + $arg = $argumentPartList[0] ?? 'NOW()'; + + return "FLOOR(EXTRACT(EPOCH FROM {$arg}))"; + } + + if ($function === 'BINARY') { + // Not supported. + return $argumentPartList[0] ?? '0'; + } + + if ($function === 'TZ') { + if (count($argumentPartList) < 2) { + throw new RuntimeException("Not enough arguments for function TZ."); + } + + $offsetHoursString = $argumentPartList[1]; + if (str_starts_with($offsetHoursString, '\'') && str_ends_with($offsetHoursString, '\'')) { + $offsetHoursString = substr($offsetHoursString, 1, -1); + } + + return "{$argumentPartList[0]} + INTERVAL 'HOUR {$offsetHoursString}'"; + } + + if ($function === 'POSITION_IN_LIST') { + if (count($argumentPartList) <= 1) { + return $this->quote(1); + } + + $field = $argumentPartList[0]; + + $pairs = array_map( + fn($i) => [$i, $argumentPartList[$i]], + array_keys($argumentPartList) + ); + + $whenParts = array_map(function ($item) use ($field) { + $resolution = intval($item[0]); + $value = $item[1]; + + return " WHEN {$field} = {$value} THEN {$resolution}"; + }, array_slice($pairs, 1)); + + return "CASE" . implode('', $whenParts) . " ELSE 0 END"; + } + + if ($function === 'IFNULL') { + $function = 'COALESCE'; + } + + if (str_starts_with($function, 'YEAR_') && $function !== 'YEAR_NUMBER') { + $fiscalShift = substr($function, 5); + + if (is_numeric($fiscalShift)) { + $fiscalShift = (int) $fiscalShift; + $fiscalFirstMonth = $fiscalShift + 1; + + return + "CASE WHEN EXTRACT(MONTH FROM {$part}) >= {$fiscalFirstMonth} THEN ". + "EXTRACT(YEAR FROM {$part}) ". + "ELSE EXTRACT(YEAR FROM {$part}) - 1 END"; + } + } + + if (str_starts_with($function, 'QUARTER_') && $function !== 'QUARTER_NUMBER') { + $fiscalShift = substr($function, 8); + + if (is_numeric($fiscalShift)) { + $fiscalShift = (int) $fiscalShift; + $fiscalFirstMonth = $fiscalShift + 1; + $fiscalDistractedMonth = $fiscalFirstMonth < 4 ? + 12 - $fiscalFirstMonth : + 12 - $fiscalFirstMonth + 1; + + return + "CASE WHEN EXTRACT(MONTH FROM {$part}) >= {$fiscalFirstMonth} " . + "THEN " . + "CONCAT(EXTRACT(YEAR FROM {$part}), '_', " . + "FLOOR((EXTRACT(MONTH FROM {$part}) - {$fiscalFirstMonth}) / 3) + 1) " . + "ELSE CONCAT(EXTRACT(YEAR FROM {$part}) - 1, '_', " . + "CEIL(EXTRACT(MONTH FROM {$part}) + {$fiscalDistractedMonth}) / 3)) " . + "END"; + } + } + + switch ($function) { + case 'MONTH': + return "TO_CHAR({$part}, 'YYYY-MM')"; + + case 'DAY': + return "TO_CHAR({$part}, 'YYYY-MM-DD')"; + + case 'WEEK': + case 'WEEK_0': + case 'WEEK_1': + return "CONCAT(TO_CHAR({$part}, 'YYYY'), '/', TRIM(LEADING '0' FROM TO_CHAR({$part}, 'IW')))"; + + case 'QUARTER': + return "CONCAT(TO_CHAR({$part}, 'YYYY'), '_', TO_CHAR({$part}, 'Q'))"; + + case 'WEEK_NUMBER_0': + case 'WEEK_NUMBER': + case 'WEEK_NUMBER_1': + // Monday week-start not implemented. + return "TO_CHAR({$part}, 'IW')::INTEGER"; + + case 'HOUR_NUMBER': + case 'HOUR': + return "EXTRACT(HOUR FROM {$part})"; + + case 'MINUTE_NUMBER': + case 'MINUTE': + return "EXTRACT(MINUTE FROM {$part})"; + + case 'SECOND_NUMBER': + case 'SECOND': + return "FLOOR(EXTRACT(SECOND FROM {$part}))"; + + case 'DATE_NUMBER': + case 'DAYOFMONTH': + return "EXTRACT(DAY FROM {$part})"; + + case 'DAYOFWEEK_NUMBER': + case 'DAYOFWEEK': + return "EXTRACT(DOW FROM {$part})"; + + case 'MONTH_NUMBER': + return "EXTRACT(MONTH FROM {$part})"; + + case 'YEAR_NUMBER': + case 'YEAR': + return "EXTRACT(YEAR FROM {$part})"; + + case 'QUARTER_NUMBER': + return "EXTRACT(QUARTER FROM {$part})"; + } + + if (str_starts_with($function, 'TIMESTAMPDIFF_')) { + $from = $argumentPartList[0] ?? $this->quote(0); + $to = $argumentPartList[1] ?? $this->quote(0); + + switch ($function) { + case 'TIMESTAMPDIFF_YEAR': + return "EXTRACT(YEAR FROM {$to} - {$from})"; + + case 'TIMESTAMPDIFF_MONTH': + return "EXTRACT(MONTH FROM {$to}) - {$from})"; + + case 'TIMESTAMPDIFF_WEEK': + return "FLOOR(EXTRACT(DAY FROM {$to} - {$from}) / 7)"; + + case 'TIMESTAMPDIFF_DAY': + return "EXTRACT(DAY FROM ({$to}) - {$from})"; + + case 'TIMESTAMPDIFF_HOUR': + return "EXTRACT(HOUR FROM {$to} - {$from})"; + + case 'TIMESTAMPDIFF_MINUTE': + return "EXTRACT(MINUTE FROM {$to} - {$from})"; + + case 'TIMESTAMPDIFF_SECOND': + return "FLOOR(EXTRACT(SECOND FROM {$to} - {$from}))"; + } + } + + return parent::getFunctionPart( + $function, + $part, + $params, + $entityType, + $distinct, + $argumentPartList + ); + } + + public function composeDelete(DeleteQuery $query): string + { + if ( + $query->getJoins() !== [] || + $query->getLeftJoins() !== [] || + $query->getLimit() !== null || + $query->getOrder() !== [] + ) { + $subQueryBuilder = SelectBuilder::create() + ->select('id') + ->from($query->getFrom()) + ->order($query->getOrder()); + + foreach ($query->getJoins() as $join) { + $subQueryBuilder->join($join); + } + + foreach ($query->getLeftJoins() as $join) { + $subQueryBuilder->leftJoin($join); + } + + if ($query->getWhere()) { + $subQueryBuilder->where($query->getWhere()); + } + + if ($query->getLimit() !== null) { + $subQueryBuilder->limit(null, $query->getLimit()); + } + + $builder = DeleteBuilder::create() + ->from($query->getFrom(), $query->getFromAlias()) + ->where( + Cond::in( + Cond::column('id'), + $subQueryBuilder->build() + ) + ); + + $query = $builder->build(); + } + + return parent::composeDelete($query); + } + + public function composeUpdate(UpdateQuery $query): string + { + if ( + $query->getJoins() !== [] || + $query->getLeftJoins() !== [] || + $query->getLimit() !== null || + $query->getOrder() !== [] + ) { + $subQueryBuilder = SelectBuilder::create() + ->select('id') + ->from($query->getIn()) + ->order($query->getOrder()) + ->forUpdate(); + + foreach ($query->getJoins() as $join) { + $subQueryBuilder->join($join); + } + + foreach ($query->getLeftJoins() as $join) { + $subQueryBuilder->leftJoin($join); + } + + if ($query->getWhere()) { + $subQueryBuilder->where($query->getWhere()); + } + + if ($query->getLimit() !== null) { + $subQueryBuilder->limit(null, $query->getLimit()); + } + + $builder = UpdateBuilder::create() + ->in($query->getIn()) + ->set($query->getSet()) + ->where( + Cond::in( + Cond::column('id'), + $subQueryBuilder->build() + ) + ); + + $query = $builder->build(); + } + + return parent::composeUpdate($query); + } + + public function composeInsert(InsertQuery $query): string + { + $params = $query->getRaw(); + $params = $this->normalizeInsertParams($params); + + $entityType = $params['into']; + $columns = $params['columns']; + $updateSet = $params['updateSet']; + + $columnsPart = $this->getInsertColumnsPart($columns); + $valuesPart = $this->getInsertValuesPart($entityType, $params); + $updatePart = $updateSet ? $this->getInsertUpdatePart($updateSet) : null; + + $table = $this->toDb($entityType); + + $sql = "INSERT INTO " . $this->quoteIdentifier($table) . " ({$columnsPart}) {$valuesPart}"; + + if ($updatePart) { + $updateColumnsPart = implode(', ', + array_map(fn ($item) => $this->quoteIdentifier($this->toDb($this->sanitize($item))), + $this->getEntityUniqueColumns($entityType) + ) + ); + + $sql .= " ON CONFLICT({$updateColumnsPart}) DO UPDATE SET " . $updatePart; + } + + return $sql; + } + + /** + * @return string[] + */ + private function getEntityUniqueColumns(string $entityType): array + { + $indexes = $this->metadata + ->getDefs() + ->getEntity($entityType) + ->getIndexList(); + + foreach ($indexes as $index) { + if ($index->isUnique()) { + return $index->getColumnList(); + } + } + + return ['id']; + } + + /** + * @param array $values + * @param array $params + */ + protected function getSetPart(Entity $entity, array $values, array $params): string + { + if (!count($values)) { + throw new RuntimeException("ORM Query: No SET values for update query."); + } + + $list = []; + + foreach ($values as $attribute => $value) { + $isNotValue = false; + + if (str_ends_with($attribute, ':')) { + $attribute = substr($attribute, 0, -1); + $isNotValue = true; + } + + if (strpos($attribute, '.') > 0) { + [$alias, $attribute] = explode('.', $attribute); + + $alias = $this->sanitize($alias); + $column = $this->toDb($this->sanitize($attribute)); + + $left = $this->quoteColumn("{$alias}.{$column}"); + } + else { + $column = $this->toDb($this->sanitize($attribute)); + + $left = $this->quoteColumn("{$column}"); // Diff. + } + + $right = $isNotValue ? + $this->convertComplexExpression($entity, $value, false, $params) : + $this->quote($value); + + $list[] = $left . " = " . $right; + } + + return implode(', ', $list); + } + + public function composeRollbackToSavepoint(string $savepointName): string + { + return 'ROLLBACK TRANSACTION TO SAVEPOINT ' . $this->sanitize($savepointName); + } + + public function composeLockTable(LockTableQuery $query): string + { + $params = $query->getRaw(); + + $table = $this->toDb($this->sanitize($params['table'])); + + $mode = $params['mode']; + + if (empty($table)) { + throw new LogicException(); + } + + if (!in_array($mode, [LockTableQuery::MODE_SHARE, LockTableQuery::MODE_EXCLUSIVE])) { + throw new LogicException(); + } + + $sql = "LOCK TABLE " . $this->quoteIdentifier($table) . " IN "; + + $modeMap = [ + LockTableQuery::MODE_SHARE => 'SHARE', + LockTableQuery::MODE_EXCLUSIVE => 'EXCLUSIVE', + ]; + + $sql .= $modeMap[$mode] . " MODE"; + + return $sql; + } + + protected function limit(string $sql, ?int $offset = null, ?int $limit = null): string + { + if (!is_null($offset) && !is_null($limit)) { + $offset = intval($offset); + $limit = intval($limit); + + $sql .= " LIMIT {$limit} OFFSET {$offset}"; + + return $sql; + } + + if (!is_null($limit)) { + $limit = intval($limit); + + $sql .= " LIMIT {$limit}"; + + return $sql; + } + + return $sql; + } +} diff --git a/application/Espo/Resources/metadata/app/databasePlatforms.json b/application/Espo/Resources/metadata/app/databasePlatforms.json index e621f98151..94b4d18dac 100644 --- a/application/Espo/Resources/metadata/app/databasePlatforms.json +++ b/application/Espo/Resources/metadata/app/databasePlatforms.json @@ -15,6 +15,12 @@ } }, "Postgresql": { - "dbalConnectionFactoryClassName": "Espo\\Core\\Utils\\Database\\Dbal\\Factories\\PostgresqlConnectionFactory" + "detailsProviderClassName": "Espo\\Core\\Utils\\Database\\DetailsProviders\\PostgresqlDetailsProvider", + "dbalConnectionFactoryClassName": "Espo\\Core\\Utils\\Database\\Dbal\\Factories\\PostgresqlConnectionFactory", + "indexHelperClassName": "Espo\\Core\\Utils\\Database\\Orm\\IndexHelpers\\PostgresqlIndexHelper", + "columnPreparatorClassName": "Espo\\Core\\Utils\\Database\\Schema\\ColumnPreparators\\PostgresqlColumnPreparator", + "dbalTypeClassNameMap": { + "uuid": "Espo\\Core\\Utils\\Database\\Dbal\\Types\\UuidType" + } } } diff --git a/application/Espo/Resources/metadata/app/orm.json b/application/Espo/Resources/metadata/app/orm.json index 59c92fe178..37b3422604 100644 --- a/application/Espo/Resources/metadata/app/orm.json +++ b/application/Espo/Resources/metadata/app/orm.json @@ -2,10 +2,15 @@ "functionConverterClassNameMap_Mysql": { "ABS": "Espo\\Core\\ORM\\QueryComposer\\Part\\FunctionConverters\\Abs" }, + "functionConverterClassNameMap_Postgresql": { + "ABS": "Espo\\Core\\ORM\\QueryComposer\\Part\\FunctionConverters\\Abs" + }, "queryComposerClassNameMap": { - "Mysql": "Espo\\ORM\\QueryComposer\\MysqlQueryComposer" + "Mysql": "Espo\\ORM\\QueryComposer\\MysqlQueryComposer", + "Postgresql": "Espo\\ORM\\QueryComposer\\PostgresqlQueryComposer" }, "pdoFactoryClassNameMap": { - "Mysql": "Espo\\ORM\\PDO\\MysqlPDOFactory" + "Mysql": "Espo\\ORM\\PDO\\MysqlPDOFactory", + "Postgresql": "Espo\\ORM\\PDO\\PostgresqlPDOFactory" } } diff --git a/tests/unit/Espo/ORM/PostgresqlQueryComposerTest.php b/tests/unit/Espo/ORM/PostgresqlQueryComposerTest.php new file mode 100644 index 0000000000..909619edaa --- /dev/null +++ b/tests/unit/Espo/ORM/PostgresqlQueryComposerTest.php @@ -0,0 +1,209 @@ +createMock(MetadataDataProvider::class); + + $metadataDataProvider + ->expects($this->any()) + ->method('get') + ->willReturn($ormMetadata); + + $metadata = new Metadata($metadataDataProvider); + + $this->queryBuilder = new QueryBuilder(); + + $pdo = $this->createMock('MockPDO'); + $pdo + ->expects($this->any()) + ->method('quote') + ->will($this->returnCallback(function() { + $args = func_get_args(); + + return "'" . $args[0] . "'"; + })); + + $this->entityManager = $this->createMock(EntityManager::class); + $entityFactory = $this->createMock(EntityFactory::class); + + $entityFactory + ->expects($this->any()) + ->method('create') + ->will( + $this->returnCallback(function () use ($metadata) { + $args = func_get_args(); + $className = "tests\\unit\\testData\\DB\\" . $args[0]; + $defs = $metadata->get($args[0]) ?? []; + + return new $className($args[0], $defs, $this->entityManager); + }) + ); + + $this->queryComposer = new QueryComposer($pdo, $entityFactory, $metadata); + + $this->post = $entityFactory->create('Post'); + $this->comment = $entityFactory->create('Comment'); + $this->tag = $entityFactory->create('Tag'); + $this->note = $entityFactory->create('Note'); + $this->contact = $entityFactory->create('Contact'); + $this->account = $entityFactory->create('Account'); + } + + public function testUpdate1(): void + { + $query = UpdateBuilder::create() + ->in('Comment') + ->set(['name' => '1']) + ->where(['name' => 'post.name']) + ->join('post') + ->limit(1) + ->order('name') + ->build(); + + $sql = $this->queryComposer->composeUpdate($query); + + $expectedSql = + 'UPDATE "comment" SET "name" = \'1\' WHERE "comment"."id" IN ' . + '(SELECT "comment"."id" AS "id" FROM "comment" ' . + 'JOIN "post" AS "post" ON "comment"."post_id" = "post"."id" ' . + 'WHERE "comment"."name" = \'post.name\' AND "comment"."deleted" = false ' . + 'ORDER BY "comment"."name" ASC LIMIT 1 OFFSET 0 FOR UPDATE)'; + + $this->assertEquals($expectedSql, $sql); + } + + public function testDelete1(): void + { + $query = DeleteBuilder::create() + ->from('Account') + ->where(['name' => 'test']) + ->build(); + + $sql = $this->queryComposer->composeDelete($query); + + $expectedSql = + "DELETE FROM \"account\" " . + "WHERE \"account\".\"name\" = 'test'"; + + $this->assertEquals($expectedSql, $sql); + } + + public function testDelete2(): void + { + $query = DeleteBuilder::create() + ->from('Comment') + ->join('post') + ->where(['name' => 'post.name']) + ->limit(1) + ->order('name') + ->build(); + + $sql = $this->queryComposer->composeDelete($query); + + $expectedSql = + 'DELETE FROM "comment" WHERE "comment"."id" IN ' . + '(SELECT "comment"."id" AS "id" FROM "comment" ' . + 'JOIN "post" AS "post" ON "comment"."post_id" = "post"."id" ' . + 'WHERE "comment"."name" = \'post.name\' AND "comment"."deleted" = false ' . + 'ORDER BY "comment"."name" ASC LIMIT 1 OFFSET 0)'; + + $this->assertEquals($expectedSql, $sql); + } + + public function testInsertUpdate1(): void + { + $query = InsertBuilder::create() + ->into('PostTag') + ->columns(['id', 'postId', 'tagId']) + ->values([ + 'id' => '1', + 'postId' => 'post-id', + 'tagId' => 'tag-id', + ]) + ->updateSet([ + 'deleted' => 0 + ]) + ->build(); + + $sql = $this->queryComposer->composeInsert($query); + + $expectedSql = + "INSERT INTO \"post_tag\" (\"id\", \"post_id\", \"tag_id\") VALUES ('1', 'post-id', 'tag-id') " . + "ON CONFLICT(\"post_id\", \"tag_id\") DO UPDATE SET \"deleted\" = 0"; + + $this->assertEquals($expectedSql, $sql); + } + + public function testInsertUpdate2(): void + { + $query = InsertBuilder::create() + ->into('Account') + ->columns(['id', 'name']) + ->values([ + 'id' => '1', + 'name' => 'name', + ]) + ->updateSet([ + 'deleted' => 0 + ]) + ->build(); + + $sql = $this->queryComposer->composeInsert($query); + + $expectedSql = + "INSERT INTO \"account\" (\"id\", \"name\") VALUES ('1', 'name') " . + "ON CONFLICT(\"id\") DO UPDATE SET \"deleted\" = 0"; + + $this->assertEquals($expectedSql, $sql); + } +} diff --git a/tests/unit/testData/DB/ormMetadata.php b/tests/unit/testData/DB/ormMetadata.php index bcc76e461b..a14ffb474f 100644 --- a/tests/unit/testData/DB/ormMetadata.php +++ b/tests/unit/testData/DB/ormMetadata.php @@ -306,6 +306,13 @@ return [ 'relations' => [ ], + 'indexes' => [ + 'postIdTagId' => [ + 'key' => 'UNIQ_POST_ID_TAG_ID', + 'type' => 'unique', + 'columns' => ['postId', 'tagId'], + ], + ], ], 'Note' => [ @@ -567,4 +574,4 @@ return [ ], ], ], -]; \ No newline at end of file +];