[SYSTEMDS-??] Modality Alignment, Contrastive Learning, PDF and Transcript Loader

.asf.yaml

@@ -0,0 +1,60 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Document for .asf.yml file is available at
+# https://github.com/apache/infrastructure-asfyaml/blob/main/README.md
+
+github:
+  description: "An open source ML system for the end-to-end data science lifecycle"
+  homepage: https://systemds.apache.org/
+  labels:
+    - systemds
+    - java
+    - python
+    - dml
+
+  features:
+    # Enable issues management
+    issues: false
+    # Enable projects for project management boards
+    projects: true
+    # Enable wiki for documentation
+    wiki: false
+
+  # Choose the type of commit merge in the PR UI
+  enabled_merge_buttons:
+    # Enable squash button
+    squash: true
+    # Enable merge button
+    merge: false
+    # Enable rebase button
+    rebase: true
+
+
+  protected_branches:
+    main:
+      # Do not allow merge commits
+      # by allowing linear history
+      required_linear_history: true
+
+staging:
+  profile: beta
+  whoami: main

.gitattributes

@@ -0,0 +1,34 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Set the default behavior to have all files normalized to Unix-style
+# line endings upon check-in.
+* text=auto
+
+# Declare files that will always have CRLF line endings on checkout.
+*.bat text eol=crlf
+
+# Denote all files that are truly binary and should not be modified.
+*.dll binary
+*.exp binary
+*.lib binary
+*.pdb binary
+*.exe binary

.github/action/Dockerfile

@@ -0,0 +1,22 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+FROM apache/systemds:testing-latest

.github/action/action.yml

@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+name: 'DockerTestContainer'
+description: 'A github action that uses a docker image containing the dependencies needed for running the testing look at /docker for more information'
+inputs:
+  test-to-run: 
+    description: 'Full package name of the test to be executed, does support wildcards *'
+    required: true
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+   - ${{ inputs.test-to-run }}

.github/dependabot.yml

@@ -0,0 +1,27 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+version: 2
+updates:   # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:       
+      interval: "daily"

.github/workflows/build-cron.yml

@@ -0,0 +1,67 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+name: Build Different Distributions
+
+on:
+  schedule:
+    - cron: '30 1 * * 3' # Wednesday at 1:30 AM UTC
+  workflow_dispatch:
+
+jobs:
+  build-cron:
+    if: github.repository == 'apache/systemds'
+    name: ${{ matrix.os }} Java ${{ matrix.java }} ${{ matrix.javadist }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [
+          ubuntu-latest, 
+          macOS-latest, 
+          windows-latest
+          ]
+        java: [
+          # '8', 
+          '17', 
+          # '16'
+          ]
+        javadist: [
+          'temurin',
+          'zulu',
+          'adopt',
+          'adopt-openj9',
+          'liberica',
+          'microsoft'
+          ]
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v6
+
+    - name: Setup Java ${{ matrix.java }} ${{ matrix.javadist }}
+      uses: actions/setup-java@v5
+      with:
+        distribution: ${{ matrix.javadist }}
+        java-version: ${{ matrix.java }}
+        cache: 'maven'
+
+    - name: Build
+      run: mvn package -B

.github/workflows/build.yml

@@ -0,0 +1,81 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+name: Build
+
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - '*.html'
+      - 'src/main/python/**'
+      - 'dev/**'
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - '*.html'
+      - 'src/main/python/**'
+      - 'dev/**'
+    branches:
+      - main
+
+jobs:
+  build:
+    name: ${{ matrix.os }} Java ${{ matrix.java }} ${{ matrix.javadist }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [
+          ubuntu-latest, 
+          macOS-latest, 
+          windows-latest
+          ]
+        java: [
+          # '8', 
+          '17', 
+          # '16'
+          ]
+        javadist: [
+          # 'temurin',
+          # 'zulu',
+          'adopt',
+          # 'adopt-openj9',
+          # 'liberica',
+          # 'microsoft'
+          ]
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v6
+
+    - name: Setup Java ${{ matrix.java }} ${{ matrix.javadist }}
+      uses: actions/setup-java@v5
+      with:
+        distribution: ${{ matrix.javadist }}
+        java-version: ${{ matrix.java }}
+        cache: 'maven'
+
+    - name: Build
+      run: mvn package -D maven.test.skip=true -B

.github/workflows/cleanup-transient-artifacts.yml

@@ -0,0 +1,54 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+name: Cleanup Transient Artifacts
+
+on: 
+  workflow_run:
+    workflows: [Java Test]
+    types: 
+      - completed
+
+jobs:
+  on-success:
+    name: Delete transient artifacts of successful workflow
+    runs-on: ubuntu-24.04
+    permissions:
+      # write permissions required for deleting artifacts
+      actions: write
+    # only run after successful jobs since some tests require retries
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v6
+
+    - name: Delete Artifacts
+      run: |
+        python .github/workflows/delete_artifacts.py \
+          -t ${{ secrets.GITHUB_TOKEN }} \
+          -a "transient_.*" \
+          -r ${{ github.event.workflow_run.id }}
+
+      env:
+        GITHUB_REPOSITORY: ${{ vars.GITHUB_REPOSITORY }}
+
+
+