add cluster-utilization report doc

Author: suiguoxin

contrib/kubespray/quick-start/services-configuration.yaml.template

@@ -225,7 +225,7 @@ authentication:
 #       smtp-auth-username: alert-sender@example.com
 #       smtp-auth-password: password-for-alert-sender
 #   cluster-utilization: # cluster-utilization is a k8s CronJob which reports the GPU utilization of the cluster
-#     # for schedule syntex, refer to https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax
+#     # for schedule syntax, refer to https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax
 #     schedule: "0 0 * * *" # daily report at UTC 00:00
 #   customized-routes:
 #     routes:

deployment/quick-start/services-configuration.yaml.template

@@ -85,7 +85,7 @@ rest-server:
 #       smtp-auth-username: alert-sender@example.com
 #       smtp-auth-password: password-for-alert-sender
 #   cluster-utilization: # cluster-utilization is a k8s CronJob which reports the GPU utilization of the cluster
-#     # for schedule syntex, refer to https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax
+#     # for schedule syntax, refer to https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax
 #     schedule: "0 0 * * *" # daily report at UTC 00:00
 #   customized-routes:
 #     routes:

docs/manual/cluster-admin/how-to-use-alert-system.md

@@ -248,3 +248,26 @@ Remember to re-build and push the docker image, and restart the `alert-manager`
 ./paictl.py config push -p /cluster-configuration -m service
 ./paictl.py service start -n alert-manager
 ```
+
+## Cluster GPU Utilization Report
+
+We provide the functionality to send cluster GPU utilization report regularly to admin users.
+
+The report includes the statistics for:
+-	Cluster GPU utilization
+-	User GPU utilization 
+-	Job GPU utilization
+
+To enable this feature, you should configure the `alert-manager` field in `services-configuration.yml`.
+`pai-bearer-token` & `cluster-utilization`->`schedule` are necessary fields for this feature.
+For the syntax of `schedule`, please refer to [Cron Schedule Syntax](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax).
+For example, `"0 0 * * *"` means daily report at UTC 00:00.
+Please also make sure that the [`email-admin`](#Existing-Actions-and-Matching-Rules) action is enabled.
+
+```yaml
+alert-manager:
+  pai-bearer-token: 'your-application-token-for-pai-rest-server'
+  cluster-utilization: # cluster-utilization is a k8s CronJob which reports the GPU utilization of the cluster
+    # for schedule syntax, refer to https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax
+    schedule: "0 0 * * *" # daily report at UTC 00:00
+```

examples/cluster-configuration/services-configuration.yaml

@@ -118,7 +118,7 @@ rest-server:
 #       smtp-auth-username: alert-sender@example.com
 #       smtp-auth-password: password-for-alert-sender
 #   cluster-utilization: # cluster-utilization is a k8s CronJob which reports the GPU utilization of the cluster
-#     # for schedule syntex, refer to https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax
+#     # for schedule syntax, refer to https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax
 #     schedule: "0 0 * * *" # daily report at UTC 00:00
 #   customized-routes:
 #     routes: