refactor: export plugin (#1460)

* refactor: export plugin

* fix: improve code

packages/core/database/src/database.ts

@@ -216,7 +216,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
     });
 
     // register database field types
-    for (const [name, field] of Object.entries(FieldTypes)) {
+    for (const [name, field] of Object.entries<any>(FieldTypes)) {
       if (['Field', 'RelationField'].includes(name)) {
         continue;
       }

packages/core/database/src/fields/array-field.ts

@@ -1,5 +1,5 @@
-import { BaseColumnFieldOptions, Field } from './field';
 import { DataTypes } from 'sequelize';
+import { BaseColumnFieldOptions, Field, ValueParser } from './field';
 
 export class ArrayField extends Field {
   get dataType() {
@@ -28,6 +28,51 @@ export class ArrayField extends Field {
     super.unbind();
     this.off('beforeSave', this.sortValue);
   }
+
+  buildValueParser(ctx: any) {
+    return new ArrayValueParser(this, ctx);
+  }
+}
+
+export class ArrayValueParser extends ValueParser {
+  async setValue(value: any) {
+    const { map, set } = this.getOptions();
+    const values = this.toArr(value);
+    if (set.size > 0) {
+      const filtered = values.map((v) => (map.has(v) ? map.get(v) : v)).filter((v) => set.has(v));
+      if (values.length === filtered.length) {
+        this.value = filtered;
+      } else {
+        this.errors.push('No matching option found');
+      }
+    } else {
+      this.value = values;
+    }
+  }
+
+  getOptions() {
+    const options = this.field.options?.['uiSchema']?.enum || [];
+    const map = new Map();
+    const set = new Set();
+    for (const option of options) {
+      set.add(option.value);
+      set.add(option.label);
+      map.set(option.label, option.value);
+    }
+    return { map, set };
+  }
+
+  toArr(value) {
+    let values = [];
+    if (!value) {
+      values = [];
+    } else if (typeof value === 'string') {
+      values = value.split(',');
+    } else if (Array.isArray(value)) {
+      values = value;
+    }
+    return values;
+  }
 }
 
 export interface ArrayFieldOptions extends BaseColumnFieldOptions {

packages/core/database/src/fields/belongs-to-field.ts

@@ -1,7 +1,8 @@
-import lodash, { omit } from 'lodash';
+import { omit } from 'lodash';
 import { BelongsToOptions as SequelizeBelongsToOptions, Utils } from 'sequelize';
 import { Reference } from '../features/ReferencesMap';
 import { checkIdentifier } from '../utils';
+import { ToOneValueParser } from './has-one-field';
 import { BaseRelationFieldOptions, RelationField } from './relation-field';
 
 export class BelongsToField extends RelationField {
@@ -115,6 +116,10 @@ export class BelongsToField extends RelationField {
     collection.model.refreshAttributes();
     // this.collection.removeIndex([this.options.foreignKey]);
   }
+
+  buildValueParser(ctx: any) {
+    return new ToOneValueParser(this, ctx);
+  }
 }
 
 export interface BelongsToFieldOptions extends BaseRelationFieldOptions, SequelizeBelongsToOptions {

packages/core/database/src/fields/belongs-to-many-field.ts

@@ -2,8 +2,10 @@ import { omit } from 'lodash';
 import { BelongsToManyOptions as SequelizeBelongsToManyOptions, Utils } from 'sequelize';
 import { Collection } from '../collection';
 import { Reference } from '../features/ReferencesMap';
+import { Repository } from '../repository';
 import { checkIdentifier } from '../utils';
 import { BelongsToField } from './belongs-to-field';
+import { ValueParser } from './field';
 import { MultipleRelationFieldOptions, RelationField } from './relation-field';
 
 export class BelongsToManyField extends RelationField {
@@ -123,6 +125,48 @@ export class BelongsToManyField extends RelationField {
     this.clearAccessors();
     delete collection.model.associations[this.name];
   }
+
+  buildValueParser(ctx: any) {
+    return new ToManyValueParser(this, ctx);
+  }
+}
+
+export class ToManyValueParser extends ValueParser {
+  async setValue(value: any) {
+    const fieldNames = this.getFileNames();
+    console.log('fieldNames', fieldNames, this.ctx.column);
+    if (this.isInterface('chinaRegion')) {
+      const repository = this.field.database.getRepository(this.field.target) as Repository;
+      this.value = await Promise.all(
+        value.split('/').map(async (v) => {
+          const instance = await repository.findOne({ filter: { [fieldNames.label]: v.trim() } });
+          return instance ? instance.get(fieldNames.value) : v;
+        }),
+      );
+    } else {
+      const dataIndex = this.ctx?.column?.dataIndex || [];
+      if (Array.isArray(dataIndex) && dataIndex.length < 2) {
+        return;
+      }
+      const field = this.ctx.column.dataIndex[1];
+      const repository = this.field.database.getRepository(this.field.target) as Repository;
+      this.value = await Promise.all(
+        value.split(',').map(async (v) => {
+          const instance = await repository.findOne({ filter: { [field]: v.trim() } });
+          return instance ? instance.get(fieldNames.value) : v;
+        }),
+      );
+    }
+  }
+
+  getFileNames() {
+    const fieldNames = this.field.options?.uiSchema?.['x-component-props']?.['fieldNames'] || {};
+    return { label: 'id', value: 'id', ...fieldNames };
+  }
+
+  isInterface(name) {
+    return this.field.options.interface === name;
+  }
 }
 
 export interface BelongsToManyFieldOptions

packages/core/database/src/fields/boolean-field.ts

@@ -1,10 +1,42 @@
 import { DataTypes } from 'sequelize';
-import { BaseColumnFieldOptions, Field } from './field';
+import { BaseColumnFieldOptions, Field, ValueParser } from './field';
 
 export class BooleanField extends Field {
   get dataType() {
     return DataTypes.BOOLEAN;
   }
+
+  buildValueParser(ctx: any) {
+    return new BooleanValueParser(this, ctx);
+  }
+}
+
+export class BooleanValueParser extends ValueParser {
+  async setValue(value: any) {
+    // Boolean
+    if (typeof value === 'boolean') {
+      this.value = value;
+    }
+    // Number
+    else if (typeof value === 'number' && [0, 1].includes(value)) {
+      this.value = value === 1;
+    }
+    // String
+    else if (typeof value === 'string') {
+      if (!value) {
+        this.value = null;
+      }
+      if (['1', 'y', 'yes', 'true', '是'].includes(value.toLowerCase())) {
+        this.value = true;
+      } else if (['0', 'n', 'no', 'false', '否'].includes(value.toLowerCase())) {
+        this.value = false;
+      } else {
+        this.errors.push(`${JSON.stringify(this.value)} value invalid`);
+      }
+    } else {
+      this.errors.push(`${JSON.stringify(this.value)} value invalid`);
+    }
+  }
 }
 
 export interface BooleanFieldOptions extends BaseColumnFieldOptions {

packages/core/database/src/fields/date-field.ts

@@ -1,10 +1,38 @@
+import { moment2str } from '@nocobase/utils';
+import moment, { isDate, isMoment } from 'moment';
 import { DataTypes } from 'sequelize';
-import { BaseColumnFieldOptions, Field } from './field';
+import { BaseColumnFieldOptions, Field, ValueParser } from './field';
 
 export class DateField extends Field {
   get dataType() {
     return DataTypes.DATE(3);
   }
+
+  buildValueParser(ctx: any) {
+    return new DateValueParser(this, ctx);
+  }
+}
+
+export class DateValueParser extends ValueParser {
+  async setValue(value: any) {
+    if (isMoment(value)) {
+      this.value = value;
+    } else if (isDate(value)) {
+      this.value = value;
+    } else if (typeof value === 'string') {
+      const props = this.getProps();
+      const m = moment(value);
+      if (m.isValid()) {
+        this.value = moment2str(m, props);
+      } else {
+        this.errors.push('Invalid date');
+      }
+    }
+  }
+
+  getProps() {
+    return this.field.options?.uiSchema?.['x-component-props'] || {};
+  }
 }
 
 export interface DateFieldOptions extends BaseColumnFieldOptions {

packages/core/database/src/fields/field.ts

@@ -40,9 +40,9 @@ export abstract class Field {
   [key: string]: any;
 
   constructor(options?: any, context?: FieldContext) {
-    this.context = context;
-    this.database = context.database;
-    this.collection = context.collection;
+    this.context = context as any;
+    this.database = this.context.database;
+    this.collection = this.context.collection;
     this.options = options || {};
     this.init();
   }
@@ -138,7 +138,7 @@ export abstract class Field {
     // 排序字段通过 sortable 控制
     const sortable = this.collection.options.sortable;
     if (sortable) {
-      let sortField: string;
+      let sortField: any;
       if (sortable === true) {
         sortField = 'sort';
       } else if (typeof sortable === 'string') {
@@ -238,4 +238,33 @@ export abstract class Field {
   typeToString() {
     return this.dataType.toString();
   }
+
+  buildValueParser(ctx: any) {
+    return new ValueParser(this, ctx);
+  }
+}
+
+export class ValueParser {
+  ctx: any;
+  field: any;
+  value: any;
+  errors = [];
+
+  constructor(field: any, ctx: any) {
+    this.field = field;
+    this.ctx = ctx;
+  }
+
+  toString() {
+    return this.value;
+  }
+
+  getValue() {
+    return this.value;
+  }
+
+  async setValue(value: any) {
+    console.log(this.field.name, value);
+    this.value = value;
+  }
 }

packages/core/database/src/fields/has-many-field.ts

@@ -5,11 +5,12 @@ import {
   ForeignKeyOptions,
   HasManyOptions,
   HasManyOptions as SequelizeHasManyOptions,
-  Utils,
+  Utils
 } from 'sequelize';
 import { Collection } from '../collection';
 import { Reference } from '../features/ReferencesMap';
 import { checkIdentifier } from '../utils';
+import { ToManyValueParser } from './belongs-to-many-field';
 import { MultipleRelationFieldOptions, RelationField } from './relation-field';
 
 export interface HasManyFieldOptions extends HasManyOptions {
@@ -185,6 +186,10 @@ export class HasManyField extends RelationField {
     // @ts-ignore
     collection.model.refreshAttributes();
   }
+
+  buildValueParser(ctx: any) {
+    return new ToManyValueParser(this, ctx);
+  }
 }
 
 export interface HasManyFieldOptions extends MultipleRelationFieldOptions, SequelizeHasManyOptions {

packages/core/database/src/fields/has-one-field.ts

@@ -1,16 +1,18 @@
-import lodash, { omit } from 'lodash';
+import { omit } from 'lodash';
 import {
   AssociationScope,
   DataType,
   ForeignKeyOptions,
   HasOneOptions,
   HasOneOptions as SequelizeHasOneOptions,
-  Utils,
+  Utils
 } from 'sequelize';
 import { Collection } from '../collection';
-import { checkIdentifier, snakeCase } from '../utils';
-import { BaseRelationFieldOptions, RelationField } from './relation-field';
 import { Reference } from '../features/ReferencesMap';
+import { Repository } from '../repository';
+import { checkIdentifier } from '../utils';
+import { ValueParser } from './field';
+import { BaseRelationFieldOptions, RelationField } from './relation-field';
 
 export interface HasOneFieldOptions extends HasOneOptions {
   /**
@@ -191,6 +193,36 @@ export class HasOneField extends RelationField {
     // @ts-ignore
     collection.model.refreshAttributes();
   }
+
+  buildValueParser(ctx: any) {
+    return new ToOneValueParser(this, ctx);
+  }
+}
+
+export class ToOneValueParser extends ValueParser {
+  async setValue(value: any) {
+    const fieldNames = this.getFileNames();
+    console.log('fieldNames', fieldNames, this.ctx.column);
+    const dataIndex = this.ctx?.column?.dataIndex || [];
+    if (Array.isArray(dataIndex) && dataIndex.length < 2) {
+      return;
+    }
+    const field = this.ctx.column.dataIndex[1];
+    const repository = this.field.database.getRepository(this.field.target) as Repository;
+    const instance = await repository.findOne({ filter: { [field]: value.trim() } });
+    if (instance) {
+      this.value = instance.get(fieldNames.value);
+    }
+  }
+
+  getFileNames() {
+    const fieldNames = this.field.options?.uiSchema?.['x-component-props']?.['fieldNames'] || {};
+    return { label: 'id', value: 'id', ...fieldNames };
+  }
+
+  isInterface(name) {
+    return this.field.options.interface === name;
+  }
 }
 
 export interface HasOneFieldOptions extends BaseRelationFieldOptions, SequelizeHasOneOptions {

packages/core/database/src/fields/number-field.ts

@@ -1,7 +1,44 @@
 import { DataTypes } from 'sequelize';
-import { BaseColumnFieldOptions, Field } from './field';
+import { BaseColumnFieldOptions, Field, ValueParser } from './field';
 
-export class IntegerField extends Field {
+abstract class NumberField extends Field {
+  buildValueParser(ctx: any) {
+    return new NumberValueParser(this, ctx);
+  }
+}
+
+function percent2float(value: string) {
+  const index = value.indexOf('.');
+  if (index === -1) {
+    return parseFloat(value) / 100;
+  }
+  const repeat = value.length - index - 2;
+  const v = parseInt('1' + '0'.repeat(repeat));
+  return (parseFloat(value) * v) / (100 * v);
+}
+
+export class NumberValueParser extends ValueParser {
+  async setValue(value: any) {
+    if (value === null || value === undefined || typeof value === 'number') {
+      this.value = value;
+    }
+    if (typeof value === 'string') {
+      if (!value) {
+        this.value = null;
+      } else if (['n/a', '-'].includes(value.toLowerCase())) {
+        this.value = null;
+      } else if (value.endsWith('%')) {
+        this.value = percent2float(value);
+        console.log(value, this.value);
+      } else {
+        const val = +value;
+        this.value = isNaN(val) ? null : val;
+      }
+    }
+  }
+}
+
+export class IntegerField extends NumberField {
   get dataType() {
     return DataTypes.INTEGER;
   }
@@ -11,7 +48,7 @@ export interface IntegerFieldOptions extends BaseColumnFieldOptions {
   type: 'integer';
 }
 
-export class BigIntField extends Field {
+export class BigIntField extends NumberField {
   get dataType() {
     return DataTypes.BIGINT;
   }
@@ -21,7 +58,7 @@ export interface BigIntFieldOptions extends BaseColumnFieldOptions {
   type: 'bigInt';
 }
 
-export class FloatField extends Field {
+export class FloatField extends NumberField {
   get dataType() {
     return DataTypes.FLOAT;
   }
@@ -31,7 +68,7 @@ export interface FloatFieldOptions extends BaseColumnFieldOptions {
   type: 'float';
 }
 
-export class DoubleField extends Field {
+export class DoubleField extends NumberField {
   get dataType() {
     return DataTypes.DOUBLE;
   }
@@ -41,7 +78,7 @@ export interface DoubleFieldOptions extends BaseColumnFieldOptions {
   type: 'double';
 }
 
-export class RealField extends Field {
+export class RealField extends NumberField {
   get dataType() {
     return DataTypes.REAL;
   }
@@ -51,7 +88,7 @@ export interface RealFieldOptions extends BaseColumnFieldOptions {
   type: 'real';
 }
 
-export class DecimalField extends Field {
+export class DecimalField extends NumberField {
   get dataType() {
     return DataTypes.DECIMAL;
   }

packages/core/database/src/fields/string-field.ts

@@ -1,10 +1,44 @@
 import { DataTypes } from 'sequelize';
-import { BaseColumnFieldOptions, Field } from './field';
+import { BaseColumnFieldOptions, Field, ValueParser } from './field';
 
 export class StringField extends Field {
   get dataType() {
     return DataTypes.STRING;
   }
+
+  buildValueParser(ctx: any) {
+    return new StringValueParser(this, ctx);
+  }
+}
+
+export class StringValueParser extends ValueParser {
+  async setValue(value: any) {
+    const { map, set } = this.getOptions();
+    if (set.size > 0) {
+      if (map.has(value)) {
+        value = map.get(value);
+      }
+      if (set.has(value)) {
+        this.value = value;
+      } else {
+        this.errors.push('No matching option found');
+      }
+    } else {
+      this.value = value;
+    }
+  }
+
+  getOptions() {
+    const options = this.field.options?.['uiSchema']?.enum || [];
+    const map = new Map();
+    const set = new Set();
+    for (const option of options) {
+      set.add(option.value);
+      set.add(option.label);
+      map.set(option.label, option.value);
+    }
+    return { map, set };
+  }
 }
 
 export interface StringFieldOptions extends BaseColumnFieldOptions {

packages/core/utils/src/date.ts

@@ -76,3 +76,65 @@ export const str2moment = (value?: string | string[], options: Str2momentOptions
     ? toMoment(value, options)
     : value;
 };
+
+const toStringByPicker = (value, picker) => {
+  if (picker === 'year') {
+    return value.format('YYYY') + '-01-01T00:00:00.000Z';
+  }
+  if (picker === 'month') {
+    return value.format('YYYY-MM') + '-01T00:00:00.000Z';
+  }
+  if (picker === 'quarter') {
+    return value.format('YYYY-MM') + '-01T00:00:00.000Z';
+  }
+  if (picker === 'week') {
+    return value.format('YYYY-MM-DD') + 'T00:00:00.000Z';
+  }
+  return value.format('YYYY-MM-DD') + 'T00:00:00.000Z';
+};
+
+const toGmtByPicker = (value: moment.Moment | moment.Moment[], picker?: any) => {
+  if (!value) {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map((val) => toStringByPicker(val, picker));
+  }
+  if (moment.isMoment(value)) {
+    return toStringByPicker(value, picker);
+  }
+};
+
+export interface Moment2strOptions {
+  showTime?: boolean;
+  gmt?: boolean;
+  picker?: 'year' | 'month' | 'week' | 'quarter';
+}
+
+export const moment2str = (value?: moment.Moment | moment.Moment[], options: Moment2strOptions = {}) => {
+  const { showTime, gmt, picker } = options;
+  if (!value) {
+    return value;
+  }
+  if (showTime) {
+    return gmt ? toGmt(value) : toLocal(value);
+  }
+  return toGmtByPicker(value, picker);
+};
+
+export const mapDateFormat = function () {
+  return (props: any) => {
+    const format = getDefaultFormat(props) as any;
+    const onChange = props.onChange;
+    return {
+      ...props,
+      format: format,
+      value: str2moment(props.value, props),
+      onChange: (value: moment.Moment | moment.Moment[]) => {
+        if (onChange) {
+          onChange(moment2str(value, props));
+        }
+      },
+    };
+  };
+};

packages/plugins/import/package.json

@@ -5,6 +5,9 @@
   "license": "AGPL-3.0",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
+  "dependencies": {
+    "xlsx": "^0.18.5"
+  },
   "devDependencies": {
     "@nocobase/client": "0.9.0-alpha.2",
     "@nocobase/test": "0.9.0-alpha.2"

packages/plugins/import/src/client/useImportAction.ts

@@ -1,14 +1,12 @@
 import { Schema, useFieldSchema, useForm } from '@formily/react';
-import { isEmpty } from '@formily/shared';
 import {
   useActionContext,
   useAPIClient,
   useBlockRequestContext,
   useCollection,
   useCollectionManager,
-  useCompile,
+  useCompile
 } from '@nocobase/client';
-import { message } from 'antd';
 import { saveAs } from 'file-saver';
 import { cloneDeep } from 'lodash';
 import { useTranslation } from 'react-i18next';
@@ -29,45 +27,43 @@ export const useDownloadXlsxTemplateAction = () => {
   const apiClient = useAPIClient();
   const actionSchema = useFieldSchema();
   const compile = useCompile();
-  const { getCollectionJoinField } = useCollectionManager();
+  const { getCollectionJoinField, getCollectionField } = useCollectionManager();
   const { name, title, getField } = useCollection();
   const { t } = useTranslation(NAMESPACE);
   const { schema: importSchema } = useImportSchema(actionSchema);
   return {
     async run() {
       const { importColumns, explain } = cloneDeep(importSchema?.['x-action-settings']?.['importSettings'] ?? {});
-      try {
-        importColumns.forEach((es) => {
-          const { uiSchema, interface: fieldInterface } =
-            getCollectionJoinField(`${name}.${es.dataIndex.join('.')}`) ?? {};
-          if (isEmpty(uiSchema) && isEmpty(fieldInterface)) {
-            throw new Error(t('Field {{fieldName}} does not exist', { fieldName: es.dataIndex.join('.') }));
+      const columns = importColumns
+        .map((column) => {
+          const field = getCollectionField(`${name}.${column.dataIndex[0]}`);
+          if (!field) {
+            return;
           }
-          es.enum = uiSchema?.enum?.map((e) => ({ value: e.value, label: e.label }));
-          if (!es.enum && uiSchema.type === 'boolean') {
-            es.enum = [
-              { value: true, label: t('Yes') },
-              { value: false, label: t('No') },
-            ];
+          if (field.interface === 'chinaRegion') {
+            column.dataIndex.push('name');
           }
-          es.defaultTitle = compile(uiSchema?.title);
-          if (fieldInterface === 'chinaRegion') {
-            es.dataIndex.push('name');
+          column.defaultTitle = compile(field?.uiSchema?.title) || field.name;
+          if (column.dataIndex.length > 1) {
+            const subField = getCollectionJoinField(`${name}.${column.dataIndex.join('.')}`);
+            if (!subField) {
+              return;
+            }
+            column.defaultTitle = column.defaultTitle + '/' + compile(subField?.uiSchema?.title) || subField.name;
           }
-        });
-      } catch (error) {
-        message.error(error.message);
-        return;
-      }
-
+          return column;
+        })
+        .filter(Boolean);
       const { data } = await resource.downloadXlsxTemplate(
         {
-          title: compile(title),
-          explain,
-          columns: JSON.stringify(compile(importColumns)),
+          values: {
+            title: compile(title),
+            explain,
+            columns: compile(columns),
+          },
         },
         {
-          method: 'get',
+          method: 'post',
           responseType: 'blob',
         },
       );
@@ -82,7 +78,7 @@ export const useImportStartAction = () => {
   const apiClient = useAPIClient();
   const actionSchema = useFieldSchema();
   const compile = useCompile();
-  const { getCollectionJoinField } = useCollectionManager();
+  const { getCollectionJoinField, getCollectionField } = useCollectionManager();
   const { name, title, getField } = useCollection();
   const { t } = useTranslation(NAMESPACE);
   const { schema: importSchema } = useImportSchema(actionSchema);
@@ -92,47 +88,47 @@ export const useImportStartAction = () => {
   return {
     async run() {
       const { importColumns, explain } = cloneDeep(importSchema?.['x-action-settings']?.['importSettings'] ?? {});
-      try {
-        importColumns.forEach((es) => {
-          const { uiSchema, interface: fieldInterface } =
-            getCollectionJoinField(`${name}.${es.dataIndex.join('.')}`) ?? {};
-          if (isEmpty(uiSchema) && isEmpty(fieldInterface)) {
-            throw new Error(t('Field {{fieldName}} does not exist', { fieldName: es.dataIndex.join('.') }));
+      const columns = importColumns
+        .map((column) => {
+          const field = getCollectionField(`${name}.${column.dataIndex[0]}`);
+          if (!field) {
+            return;
           }
-          es.enum = uiSchema?.enum?.map((e) => ({ value: e.value, label: e.label }));
-          if (!es.enum && uiSchema.type === 'boolean') {
-            es.enum = [
-              { value: true, label: t('Yes') },
-              { value: false, label: t('No') },
-            ];
+          if (field.interface === 'chinaRegion') {
+            column.dataIndex.push('name');
           }
-          es.defaultTitle = compile(uiSchema?.title);
-          if (fieldInterface === 'chinaRegion') {
-            es.dataIndex.push('name');
+          column.defaultTitle = compile(field?.uiSchema?.title) || field.name;
+          if (column.dataIndex.length > 1) {
+            const subField = getCollectionJoinField(`${name}.${column.dataIndex.join('.')}`);
+            if (!subField) {
+              return;
+            }
+            column.defaultTitle = column.defaultTitle + '/' + compile(subField?.uiSchema?.title) || subField.name;
           }
-        });
-      } catch (error) {
-        message.error(error.message);
-        return;
-      }
+          return column;
+        })
+        .filter(Boolean);
       let formData = new FormData();
       const uploadFiles = form.values.upload.map((f) => f.originFileObj);
       console.log(form, uploadFiles);
       formData.append('file', uploadFiles[0]);
-      formData.append('columns', JSON.stringify(importColumns));
+      formData.append('columns', JSON.stringify(columns));
       formData.append('explain', explain);
       setVisible(false);
       setImportModalVisible(true);
       setImportStatus(ImportStatus.IMPORTING);
-      const { data }: any = await apiClient.axios
-        .post(`${name}:importXlsx`, formData, {
+      try {
+        const { data }: any = await apiClient.axios.post(`${name}:importXlsx`, formData, {
           timeout: 10 * 60 * 1000,
-        })
-        .catch((err) => {});
-      setImportResult(data);
-      form.reset();
-      await service?.refresh?.();
-      setImportStatus(ImportStatus.IMPORTED);
+        });
+        setImportResult(data);
+        form.reset();
+        await service?.refresh?.();
+        setImportStatus(ImportStatus.IMPORTED);
+      } catch (error) {
+        setImportModalVisible(false);
+        setVisible(true);
+      }
     },
   };
 };

packages/plugins/import/src/server/actions/downloadXlsxTemplate.ts

@@ -2,14 +2,14 @@ import { Context, Next } from '@nocobase/actions';
 import xlsx from 'node-xlsx';
 
 export async function downloadXlsxTemplate(ctx: Context, next: Next) {
-  let { columns, explain, title } = ctx.action.params;
+  let { columns, explain, title } = ctx.request.body;
   if (typeof columns === 'string') {
     columns = JSON.parse(columns);
   }
   const header = columns?.map((column) => column.defaultTitle);
   const data = [header];
   if (explain?.trim() !== '') {
-    data.push([explain]);
+    data.unshift([explain]);
   }
 
   ctx.body = xlsx.build([

packages/plugins/import/src/server/actions/importXlsx.ts

@@ -1,109 +1,163 @@
 import { Context, Next } from '@nocobase/actions';
-import { Repository } from '@nocobase/database';
-import { cloneDeep } from 'lodash';
+import { Collection, Repository } from '@nocobase/database';
 import xlsx from 'node-xlsx';
-import { transform } from '../utils';
+import XLSX from 'xlsx';
+import { namespace } from '../../';
 
 const IMPORT_LIMIT_COUNT = 10000;
 
-export async function importXlsx(ctx: Context, next: Next) {
-  let { columns } = ctx.request.body as any;
-  const { ['file']: file } = ctx;
-  const { resourceName, resourceOf } = ctx.action;
-  if (typeof columns === 'string') {
-    columns = JSON.parse(columns);
-  }
-  const repository = ctx.db.getRepository<any>(resourceName, resourceOf) as Repository;
-  const collection = repository.collection;
+class Importer {
+  repository: Repository;
+  collection: Collection;
+  columns: any[];
+  items: any[][] = [];
+  headerRow;
+  context: Context;
 
-  columns = columns?.filter((col) => col?.dataIndex?.length > 0);
-  const collectionFields = columns.map((col) => collection.fields.get(col.dataIndex[0]));
-  const {
-    0: { data: originalList },
-  } = xlsx.parse(file.buffer);
-  const failureData = originalList.splice(IMPORT_LIMIT_COUNT + 1);
-  const titles = originalList.shift();
-  const legalList: any[] = [];
-  if (originalList.length > 0 && titles?.length === columns.length) {
-    // const results = (
-    //   await Promise.allSettled<any>(
-    //     originalList.map(async (item) => {
-    //       try {
-    //         const transformResult = await transform({ ctx, record: item, columns, fields: collectionFields });
-    //         legalList.push(cloneDeep(item));
-    //         return transformResult;
-    //       } catch (error) {
-    //         failureData.unshift([...item, error.message]);
-    //       }
-    //     }),
-    //   )
-    // ).filter((item) => 'value' in item && item.value !== undefined);
-    const values: any[] = [];
-    for (const item of originalList) {
-      try {
-        const transformResult = await transform({ ctx, record: item, columns, fields: collectionFields });
-        values.push(transformResult);
-        legalList.push(cloneDeep<any>(item));
-      } catch (error) {
-        failureData.unshift([...item, error.message]);
+  constructor(ctx: Context) {
+    const { resourceName, resourceOf } = ctx.action;
+    this.context = ctx;
+    this.repository = ctx.db.getRepository<any>(resourceName, resourceOf);
+    this.collection = this.repository.collection;
+    this.parseXlsx();
+  }
+
+  getRows() {
+    const workbook = XLSX.read(this.context.file.buffer, {
+      type: 'buffer',
+      // cellDates: true,
+      // raw: false,
+    });
+    const r = workbook.Sheets[workbook.SheetNames[0]];
+    const rows = XLSX.utils.sheet_to_json<any>(r, { header: 1, defval: null, raw: false });
+    return rows;
+  }
+
+  parseXlsx() {
+    const rows = this.getRows();
+    let columns = this.context.request.body.columns as any[];
+    if (typeof columns === 'string') {
+      columns = JSON.parse(columns);
+    }
+    this.columns = columns.map((column) => {
+      return {
+        ...column,
+        field: this.collection.fields.get(column.dataIndex[0]),
+      };
+    });
+    const str = this.columns.map((column) => column.defaultTitle).join('||');
+    for (const row of rows) {
+      if (this.hasHeaderRow()) {
+        if (row && row.join('').trim()) {
+          this.items.push(row);
+        }
+      }
+      if (str === row.filter((r) => r).join('||')) {
+        this.headerRow = row;
       }
     }
-    //@ts-ignore
-    // const values = results.map((r) => r.value);
-    const result = await ctx.db.sequelize.transaction(async (transaction) => {
-      let sort: number = 0;
-      if (collection.options.sortable) {
-        sort = await repository.model.max<number, any>('sort', { transaction });
-      }
-      for (const [index, val] of values.entries()) {
-        if (val === undefined || val === null) {
+  }
+
+  getFieldByIndex(index) {
+    return this.columns[index].field;
+  }
+
+  async getItems() {
+    const items: any[] = [];
+    for (const row of this.items) {
+      const values = {};
+      const errors = [];
+      for (let index = 0; index < row.length; index++) {
+        if (!this.columns[index]) {
           continue;
         }
+        const column = this.columns[index];
+        const { field, defaultTitle } = column;
+        let value = row[index];
+        if (value === undefined || value === null) {
+          continue;
+        }
+        const parser = field.buildValueParser({ ...this.context, column });
+        await parser.setValue(typeof value === 'string' ? value.trim() : value);
+        value = parser.getValue();
+        if (parser.errors.length > 0) {
+          errors.push(`${defaultTitle}: ${parser.errors.join(';')}`);
+        }
+        if (value === undefined) {
+          continue;
+        }
+        values[field.name] = value;
+      }
+      items.push({
+        row,
+        values,
+        errors,
+      });
+    }
+    return items;
+  }
+
+  hasSortField() {
+    return !!this.collection.options.sortable;
+  }
+
+  async run() {
+    return await this.context.db.sequelize.transaction(async (transaction) => {
+      let sort: number = 0;
+      if (this.hasSortField()) {
+        sort = await this.repository.model.max<number, any>('sort', { transaction });
+      }
+      const result: any = [[], []];
+      for (const { row, values, errors } of await this.getItems()) {
+        if (errors.length > 0) {
+          row.push(errors.join(';'));
+          result[1].push(row);
+          continue;
+        }
+        if (this.hasSortField()) {
+          values['sort'] = ++sort;
+        }
         try {
-          let values = { ...val };
-          if (collection.options.sortable) {
-            sort += 1;
-            values['sort'] = sort;
-          }
-          await repository.create({
+          const instance = await this.repository.create({
             values,
             transaction,
             logging: false,
+            context: this.context,
           });
+          result[0].push(instance);
         } catch (error) {
-          const failData = legalList[index];
-          failData.push(error?.original?.message ?? error.message);
-          failureData.unshift(failData);
+          this.context.log.error(error, row);
+          result[1].push(row);
         }
       }
-      return {
-        successCount: originalList.length - failureData.length,
-        failureCount: failureData.length,
-      };
+      return result;
     });
-    const header = columns?.map((column) => column.defaultTitle);
-    ctx.body = {
-      rows: xlsx.build([
-        {
-          name: file.originalname,
-          data: [header].concat(failureData),
-        },
-      ]),
-      ...result,
-    };
-  } else {
-    ctx.body = {
-      rows: file.buffer.toJSON(),
-      successCount: 0,
-      failureCount: originalList?.length ?? 0,
-    };
   }
 
-  ctx.set({
-    'Content-Type': 'application/octet-stream',
-    // to avoid "invalid character" error in header (RFC)
-    'Content-Disposition': `attachment; filename=${encodeURI('testTitle')}.xlsx`,
-  });
+  hasHeaderRow() {
+    return !!this.headerRow;
+  }
+}
+
+export async function importXlsx(ctx: Context, next: Next) {
+  const importer = new Importer(ctx);
+
+  if (!importer.hasHeaderRow()) {
+    ctx.throw(400, ctx.t('Imported template does not match, please download again.', { ns: namespace }));
+  }
+
+  const [success, failure] = await importer.run();
+
+  ctx.body = {
+    rows: xlsx.build([
+      {
+        name: ctx.file.originalname,
+        data: [importer.headerRow].concat(failure),
+      },
+    ]),
+    successCount: success.length,
+    failureCount: failure.length,
+  };
 
   await next();
 }

packages/plugins/import/src/server/locale/zh-CN.ts

@@ -7,4 +7,5 @@ export default {
   'Incorrect date format': '日期格式不正确',
   'Incorrect email format': '邮箱格式不正确',
   'Illegal percentage format': '百分比格式有误',
+  'Imported template does not match, please download again.': '导入模板不匹配，请检查导入文件标题行或重新下载导入模板'
 };