diff --git a/.gitmodules b/.gitmodules
index ba725af1588a..b9526991c258 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -25,3 +25,6 @@
 [submodule "3rdparty/cutlass_fpA_intB_gemm"]
 	path = 3rdparty/cutlass_fpA_intB_gemm
 	url = https://github.com/tlc-pack/cutlass_fpA_intB_gemm
+[submodule "3rdparty/libflash_attn"]
+	path = 3rdparty/libflash_attn
+	url = https://github.com/tlc-pack/libflash_attn
diff --git a/3rdparty/libflash_attn b/3rdparty/libflash_attn
new file mode 160000
index 000000000000..58b343e57571
--- /dev/null
+++ b/3rdparty/libflash_attn
@@ -0,0 +1 @@
+Subproject commit 58b343e57571fe5e0a5b43b5eb721acef8b35dff
diff --git a/LICENSE b/LICENSE
index 93bcbc63497e..1d26fab957c8 100644
--- a/LICENSE
+++ b/LICENSE
@@ -214,7 +214,7 @@ Apache Software Foundation License 2.0
 3rdparty/OpenCL-Headers
 3rdparty/mlperftiny
 3rdparty/nvbench (with LLVM exception)
-
+3rdparty/cutlass_fpA_intB_gemm
 
 BSD 2-clause License
 --------------------
@@ -247,3 +247,4 @@ BSD 3-Clause "New" or "Revised" License
 
 3rdparty/cutlass
 3rdparty/libbacktrace
+3rdparty/libflash_attn
diff --git a/licenses/LICENSE.libflash_attn.txt b/licenses/LICENSE.libflash_attn.txt
new file mode 100644
index 000000000000..5860e4b33f3d
--- /dev/null
+++ b/licenses/LICENSE.libflash_attn.txt
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, the respective contributors, as shown by the AUTHORS file.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/python/setup.py b/python/setup.py
index 0f1d0b44363a..736aeab78fb7 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -103,6 +103,12 @@ def get_lib_path():
                                 candidate_path, "cutlass_fpA_intB_gemm", "cutlass", "media"
                             )
                         )
+                        _remove_path(
+                            os.path.join(candidate_path, "libflash_attn", "cutlass", "docs")
+                        )
+                        _remove_path(
+                            os.path.join(candidate_path, "libflash_attn", "cutlass", "media")
+                        )
                     break
     else:
         libs = None