v0.97/roadmap.html



<!DOCTYPE html>

<html lang="en-US">
<head>
  <meta charset="UTF-8">
  <meta http-equiv="X-UA-Compatible" content="IE=Edge">

  <link rel="stylesheet" href="/v0.97/assets/css/just-the-docs-default.css">

  
    <script src="/v0.97/assets/js/vendor/lunr.min.js"></script>
  

  <script src="/v0.97/assets/js/just-the-docs.js"></script>

  <meta name="viewport" content="width=device-width, initial-scale=1">

  
    <link rel="icon" href="/v0.97/favicon.ico" type="image/x-icon">
  

  <!-- Begin Jekyll SEO tag v2.7.1 -->
<title>Roadmap | lakeFS</title>
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Roadmap" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="New features and improvements lined-up for lakeFS. Become part of building the lakeFS roadmap." />
<meta property="og:description" content="New features and improvements lined-up for lakeFS. Become part of building the lakeFS roadmap." />
<meta property="og:site_name" content="lakeFS" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Roadmap" />
<script type="application/ld+json">
{"description":"New features and improvements lined-up for lakeFS. Become part of building the lakeFS roadmap.","@type":"WebPage","headline":"Roadmap","url":"/v0.97/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"/v0.97/assets/logo.svg"}},"@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->


  <!-- Google Tag Manager Head -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
            new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
        j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
        'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
    })(window,document,'script','dataLayer','GTM-KTZBZW9');</script>
<!-- End Google Tag Manager Head -->
<meta name="robots" content="noindex">
<meta property="og:image" content="/v0.97/assets/img/shared-image.png">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.2.1/css/fontawesome.min.css" integrity="sha512-giQeaPns4lQTBMRpOOHsYnGw1tGVzbAIHUyHRgn7+6FmiEgGGjaG0T2LZJmAPMzRCl+Cug0ItQ2xDZpTmEc+CQ==" crossorigin="anonymous" referrerpolicy="no-referrer" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.2.1/css/all.min.css" integrity="sha512-MV7K8+y+gLIBoVD59lQIYicR65iaqukzvf/nwasF0nqhPay5w/9lJmVM2hMDcnK1OnMGCdVK+iQrJ7lzPJQd1w==" crossorigin="anonymous" referrerpolicy="no-referrer" />
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.1/jquery.min.js" crossorigin="anonymous"></script>
<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.13.2/themes/smoothness/jquery-ui.css">
<script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.13.2/jquery-ui.min.js"></script>
<script src="/v0.97/assets/js/copy-code.js"></script>
<script>
    $(function () {
        $(".tabs").tabs();
    });
</script>
<script src="/v0.97/assets/js/feedback.js"></script>
<link href="https://data-folks.masto.host/@lakeFS" rel="me">

</head>

<script src="https://unpkg.com/mermaid@9.3.0/dist/mermaid.min.js"></script>
<script>
    $(document).ready(function () {
        mermaid.initialize({
            startOnLoad:true,
            theme: "default",
        });
        window.mermaid.init(undefined, document.querySelectorAll('.language-mermaid'));
    });
</script>
<body>
<!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KTZBZW9"
                  height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) -->
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
    <symbol id="svg-link" viewBox="0 0 24 24">
        <title>Link</title>
        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none"
             stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"
             class="feather feather-link">
            <path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
            <path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
        </svg>
    </symbol>
    <symbol id="svg-search" viewBox="0 0 24 24">
        <title>Search</title>
        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none"
             stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"
             class="feather feather-search">
            <circle cx="11" cy="11" r="8"></circle>
            <line x1="21" y1="21" x2="16.65" y2="16.65"></line>
        </svg>
    </symbol>
    <symbol id="svg-menu" viewBox="0 0 24 24">
        <title>Menu</title>
        <svg xmlns="http://www.w3.org/2000/svg" width="20px" height="16px" viewBox="0 0 20 16" version="1.1"
             class="feather feather-menu">
            <g id="Symbols" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
                <g id="header-/-mobile" transform="translate(-30.000000, -32.000000)" fill="#279890">
                    <g id="Group" transform="translate(30.000000, 32.000000)">
                        <path
                                d="M20,14 L20,16 L0,16 L0,14 L20,14 Z M20,7 L20,9 L0,9 L0,7 L20,7 Z M20,0 L20,2 L0,2 L0,0 L20,0 Z"
                                id="mobile-menu" />
                    </g>
                </g>
            </g>
        </svg>
    </symbol>
    <symbol id="svg-arrow-right" viewBox="0 0 24 24">
        <title>Expand</title>
        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none"
             stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"
             class="feather feather-chevron-right">
            <polyline points="9 18 15 12 9 6"></polyline>
        </svg>
    </symbol>
    <symbol id="svg-doc" viewBox="0 0 24 24">
        <title>Document</title>
        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none"
             stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"
             class="feather feather-file">
            <path d="M13 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V9z"></path>
            <polyline points="13 2 13 9 20 9"></polyline>
        </svg>
    </symbol>
</svg>
<div class="body-wrapper">
    <div class="side-bar">
        <div class="site-header">
            <a href="https://lakefs.io" class="site-title lh-tight">
  <div class="site-logo"></div>

</a>
            <a href="#" id="menu-button" class="site-button">
                <svg viewBox="0 0 24 24" class="icon">
                    <use xlink:href="#svg-menu"></use>
                </svg>
            </a>
        </div>
        <nav role="navigation" aria-label="Main" id="site-nav" class="site-nav">
            
            <div class="search">
                <div class="search-input-wrap">
                    <input type="text" id="search-input" class="search-input" tabindex="0"
                           placeholder="Search lakeFS" aria-label="Search lakeFS"
                           autocomplete="off">
                    <label for="search-input" class="search-label"><svg viewBox="0 0 24 24" class="search-icon">
                            <use xlink:href="#svg-search"></use>
                        </svg></label>
                </div>
                <div id="search-results" class="search-results"></div>
            </div>
            

            <div class="nav-category nav-version">
                <label for="selectversion">Version:</label>
                <select id="selectversion" name="version" onchange="javascript:location.href = '/' + encodeURI(this.value);">
                    <option value="" selected>Latest</option>
                </select>
            </div>
            <script async>
                window.addEventListener("load",  async () => {
                    const pathFirstLevel = location.pathname.split('/')[1];
                    const selectedVersion = pathFirstLevel.startsWith('v') && pathFirstLevel || '';
                    const selectVersionElmFirst = document.getElementById('selectversion').firstElementChild;
                    const response = await fetch('/versions.json');
                    if (!response.ok) {
                        return
                    }
                    const versions = await response.json();
                    for (let [key, value] of Object.entries(versions)) {
                        const el = document.createElement("option");
                        el.value = key;
                        el.textContent = value;
                        if (key === selectedVersion) {
                            el.selected = true;
                        }
                        selectVersionElmFirst.after(el);
                    }
                });
            </script>

            <ul class="nav-list"><li
            class="nav-list-item"><a href="/v0.97/"
           class="nav-list-link">What is lakeFS</a></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/quickstart/"
           class="nav-list-link">Quickstart</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/quickstart/run.html"
                   class="nav-list-link">Run lakeFS</a></li><li class="nav-list-item "><a href="/v0.97/quickstart/repository.html"
                   class="nav-list-link">Create a Repository</a></li><li class="nav-list-item "><a href="/v0.97/quickstart/add_data.html"
                   class="nav-list-link">Add Data</a></li><li class="nav-list-item "><a href="/v0.97/quickstart/first_commit.html"
                   class="nav-list-link">Commit the Changes</a></li><li class="nav-list-item "><a href="/v0.97/quickstart/more_quickstart_options.html"
                   class="nav-list-link">More Quickstart Options</a></li></ul></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/deploy/"
           class="nav-list-link">Deploy and Setup lakeFS</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/deploy/aws.html"
                   class="nav-list-link">AWS</a></li><li class="nav-list-item "><a href="/v0.97/deploy/azure.html"
                   class="nav-list-link">Azure</a></li><li class="nav-list-item "><a href="/v0.97/deploy/gcp.html"
                   class="nav-list-link">GCP</a></li><li class="nav-list-item "><a href="/v0.97/deploy/onprem.html"
                   class="nav-list-link">On-Prem</a></li></ul></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/integrations/"
           class="nav-list-link">Integrations</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/integrations/spark.html"
                   class="nav-list-link">Spark</a></li><li class="nav-list-item "><a href="/v0.97/integrations/delta.html"
                   class="nav-list-link">Delta Lake</a></li><li class="nav-list-item "><a href="/v0.97/integrations/aws_cli.html"
                   class="nav-list-link">AWS CLI</a></li><li class="nav-list-item "><a href="/v0.97/integrations/airflow.html"
                   class="nav-list-link">Airflow</a></li><li class="nav-list-item "><a href="/v0.97/integrations/python.html"
                   class="nav-list-link">Python</a></li><li class="nav-list-item "><a href="/v0.97/integrations/duckdb.html"
                   class="nav-list-link">DuckDB</a></li><li class="nav-list-item "><a href="/v0.97/integrations/glue_hive_metastore.html"
                   class="nav-list-link">Glue / Hive metastore</a></li><li class="nav-list-item "><a href="/v0.97/integrations/presto_trino.html"
                   class="nav-list-link">Presto/Trino</a></li><li class="nav-list-item "><a href="/v0.97/integrations/athena.html"
                   class="nav-list-link">Amazon Athena</a></li><li class="nav-list-item "><a href="/v0.97/integrations/kubeflow.html"
                   class="nav-list-link">Kubeflow</a></li><li class="nav-list-item "><a href="/v0.97/integrations/airbyte.html"
                   class="nav-list-link">Airbyte</a></li><li class="nav-list-item "><a href="/v0.97/integrations/dbt.html"
                   class="nav-list-link">dbt</a></li><li class="nav-list-item "><a href="/v0.97/integrations/kakfa.html"
                   class="nav-list-link">Kafka</a></li><li class="nav-list-item "><a href="/v0.97/integrations/sagemaker.html"
                   class="nav-list-link">SageMaker</a></li><li class="nav-list-item "><a href="/v0.97/integrations/hive.html"
                   class="nav-list-link">Hive</a></li><li class="nav-list-item "><a href="/v0.97/integrations/dremio.html"
                   class="nav-list-link">Dremio</a></li><li class="nav-list-item "><a href="/v0.97/integrations/iceberg.html"
                   class="nav-list-link">Iceberg</a></li></ul></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/hooks/"
           class="nav-list-link">Hooks</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/hooks/overview.html"
                   class="nav-list-link">Overview</a></li><li class="nav-list-item "><a href="/v0.97/hooks/lua.html"
                   class="nav-list-link">Lua</a></li><li class="nav-list-item "><a href="/v0.97/hooks/webhooks.html"
                   class="nav-list-link">Webhooks</a></li><li class="nav-list-item "><a href="/v0.97/hooks/airflow.html"
                   class="nav-list-link">Airflow</a></li></ul></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/use_cases/"
           class="nav-list-link">Use Cases</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/use_cases/etl_testing.html"
                   class="nav-list-link">ETL Testing Environment</a></li><li class="nav-list-item "><a href="/v0.97/use_cases/rollback.html"
                   class="nav-list-link">Rollback</a></li><li class="nav-list-item "><a href="/v0.97/use_cases/reproducibility.html"
                   class="nav-list-link">Reproducibility</a></li><li class="nav-list-item "><a href="/v0.97/use_cases/cicd_for_data.html"
                   class="nav-list-link">CI/CD for data lakes</a></li></ul></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/howto/"
           class="nav-list-link">How-To</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/howto/import.html"
                   class="nav-list-link">Import data into lakeFS</a></li><li class="nav-list-item "><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                        <use xlink:href="#svg-arrow-right"></use>
                    </svg></a><a href="/v0.97/howto/garbage-collection-index.html"
                   class="nav-list-link">Garbage Collection</a><ul class="nav-list"><li class="nav-list-item ">
                        <a href="/v0.97/howto/garbage-collection-committed.html"
                           class="nav-list-link">Committed Objects</a>
                    </li><li class="nav-list-item ">
                        <a href="/v0.97/howto/garbage-collection-uncommitted.html"
                           class="nav-list-link">Uncommitted Objects</a>
                    </li><li class="nav-list-item ">
                        <a href="/v0.97/howto/gc-internals.html"
                           class="nav-list-link">Committed GC Internals</a>
                    </li></ul></li><li class="nav-list-item "><a href="/v0.97/howto/upgrade.html"
                   class="nav-list-link">Upgrade lakeFS</a></li><li class="nav-list-item "><a href="/v0.97/howto/export.html"
                   class="nav-list-link">Exporting Data</a></li><li class="nav-list-item "><a href="/v0.97/howto/copying.html"
                   class="nav-list-link">Copying data to/from lakeFS</a></li><li class="nav-list-item "><a href="/v0.97/howto/protect-branches.html"
                   class="nav-list-link">Protect Branches</a></li><li class="nav-list-item "><a href="/v0.97/howto/migrate-away.html"
                   class="nav-list-link">Migrating away from lakeFS</a></li><li class="nav-list-item "><a href="/v0.97/howto/sizing-guide.html"
                   class="nav-list-link">Sizing Guide</a></li></ul></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/reference/"
           class="nav-list-link">Reference</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/reference/configuration.html"
                   class="nav-list-link">Server Configuration</a></li><li class="nav-list-item "><a href="/v0.97/reference/cli.html"
                   class="nav-list-link">lakectl (lakeFS command-line tool)</a></li><li class="nav-list-item "><a href="/v0.97/reference/api.html"
                   class="nav-list-link">API Reference</a></li><li class="nav-list-item "><a href="/v0.97/reference/spark-client.html"
                   class="nav-list-link">Spark Client</a></li><li class="nav-list-item "><a href="/v0.97/reference/s3.html"
                   class="nav-list-link">S3 Supported API</a></li><li class="nav-list-item "><a href="/v0.97/reference/authentication.html"
                   class="nav-list-link">Authentication</a></li><li class="nav-list-item "><a href="/v0.97/reference/saml-authentication-integration.html"
                   class="nav-list-link">SAML Authentication Integration</a></li><li class="nav-list-item "><a href="/v0.97/reference/remote-authenticator.html"
                   class="nav-list-link">Remote Authenticator</a></li><li class="nav-list-item "><a href="/v0.97/reference/rbac.html"
                   class="nav-list-link">Role-Based Access Control (RBAC)</a></li><li class="nav-list-item "><a href="/v0.97/reference/monitor.html"
                   class="nav-list-link">Monitoring using Prometheus</a></li><li class="nav-list-item "><a href="/v0.97/reference/access-control-lists.html"
                   class="nav-list-link">Access Control Lists (ACL)</a></li><li class="nav-list-item "><a href="/v0.97/reference/audit.html"
                   class="nav-list-link">Audit Logs</a></li><li class="nav-list-item "><a href="/v0.97/reference/presigned-url.html"
                   class="nav-list-link">Presigned URL</a></li></ul></li><li
            class="nav-list-item"><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                <use xlink:href="#svg-arrow-right"></use>
            </svg></a><a href="/v0.97/understand/"
           class="nav-list-link">Understanding lakeFS</a><ul class="nav-list "><li class="nav-list-item "><a href="/v0.97/understand/architecture.html"
                   class="nav-list-link">Architecture</a></li><li class="nav-list-item "><a href="/v0.97/understand/model.html"
                   class="nav-list-link">Model</a></li><li class="nav-list-item "><a href="/v0.97/understand/performance-best-practices.html"
                   class="nav-list-link">Performance Best Practices</a></li><li class="nav-list-item "><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                        <use xlink:href="#svg-arrow-right"></use>
                    </svg></a><a href="/v0.97/understand/how/"
                   class="nav-list-link">How it Works</a><ul class="nav-list"><li class="nav-list-item ">
                        <a href="/v0.97/understand/how/versioning-internals.html"
                           class="nav-list-link">Versioning Internals</a>
                    </li><li class="nav-list-item ">
                        <a href="/v0.97/understand/how/kv.html"
                           class="nav-list-link">Internal database structure</a>
                    </li><li class="nav-list-item ">
                        <a href="/v0.97/understand/how/merge.html"
                           class="nav-list-link">Merge</a>
                    </li></ul></li><li class="nav-list-item "><a href="#" class="nav-list-expander"><svg viewBox="0 0 24 24">
                        <use xlink:href="#svg-arrow-right"></use>
                    </svg></a><a href="/v0.97/understand/data_lifecycle_management/"
                   class="nav-list-link">Data Lifecycle Management</a><ul class="nav-list"><li class="nav-list-item ">
                        <a href="/v0.97/understand/data_lifecycle_management/data-devenv.html"
                           class="nav-list-link">In Test</a>
                    </li><li class="nav-list-item ">
                        <a href="/v0.97/understand/data_lifecycle_management/ci.html"
                           class="nav-list-link">During Deployment</a>
                    </li><li class="nav-list-item ">
                        <a href="/v0.97/understand/data_lifecycle_management/production.html"
                           class="nav-list-link">In Production</a>
                    </li></ul></li><li class="nav-list-item "><a href="/v0.97/understand/glossary.html"
                   class="nav-list-link">Glossary</a></li></ul></li><li
            class="nav-list-item active"><a href="/v0.97/roadmap.html"
           class="nav-list-link active">Roadmap</a></li><li
            class="nav-list-item"><a href="/v0.97/commitment.html"
           class="nav-list-link">Commitment to OSS</a></li><li
            class="nav-list-item"><a href="/v0.97/contributing.html"
           class="nav-list-link">Contributing</a></li><li
            class="nav-list-item"><a href="/v0.97/faq.html"
           class="nav-list-link">FAQ</a></li></ul>
            <div class="mobile-menu">
                
<nav aria-label="Auxiliary" class="aux-nav">
    <ul class="aux-nav-list">
        
        
        <li class="aux-nav-list-item">
            <a href="https://docs.lakefs.io/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                Docs
            </a>
        </li>
        
        <li class="aux-nav-list-item">
            <a href="https://lakefs.io/blog/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                Blog
            </a>
        </li>
        
        <li class="aux-nav-list-item">
            <a href="https://lakefs.io/community/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                Community
            </a>
        </li>
        
        <li class="aux-nav-list-item">
            <a href="https://github.com/treeverse/lakeFS" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                GitHub
            </a>
        </li>
        
        
        <li class="aux-nav-list-item button">
            <a href="https://lakefs.io/cloud-registration/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                lakeFS Cloud
            </a>
        </li>
        
        
    </ul>
</nav>

            </div>
        </nav>

    </div>
    <div class="main" id="top">
         <div class="feedback-container">
            <div id="is-helpful-ty">
                <div class="text-epsilon">Thank you for your feedback.</div>
                <div class="mt-2 text-epsilon">
                    <a href="https://lakefs.io/slack" target="_blank">Join the community</a> to get more help.
                </div>
            </div>
            <div class="feedback-buttons">
                <span class="tooltip">
                    <button class="page-helpful-btn far fa-lg fa-thumbs-up" id="page-helpful-yes"></button>
                    <span class="tooltiptext">
                        This page is <b>helpful</b>
                    </span>
                </span>
                <span class="tooltip">
                    <button class="page-helpful-btn far fa-lg fa-thumbs-down" id="page-helpful-no"></button>
                    <span class="tooltiptext">
                        This page is <b>not helpful</b>
                    </span>
                </span>
            </div>
        </div>
        <div id="main-header" class="main-header">
            
<nav aria-label="Auxiliary" class="aux-nav">
    <ul class="aux-nav-list">
        
        
        <li class="aux-nav-list-item">
            <a href="https://docs.lakefs.io/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                Docs
            </a>
        </li>
        
        <li class="aux-nav-list-item">
            <a href="https://lakefs.io/blog/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                Blog
            </a>
        </li>
        
        <li class="aux-nav-list-item">
            <a href="https://lakefs.io/community/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                Community
            </a>
        </li>
        
        <li class="aux-nav-list-item">
            <a href="https://github.com/treeverse/lakeFS" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                GitHub
            </a>
        </li>
        
        
        <li class="aux-nav-list-item button">
            <a href="https://lakefs.io/cloud-registration/" class="site-button"  target="_blank"
               rel="noopener noreferrer" >
                lakeFS Cloud
            </a>
        </li>
        
        
    </ul>
</nav>

        </div>
        <div id="main-content-wrap" class="main-content-wrap">
            
            
            <div id="main-content" class="main-content" role="main">
                
                <h1 class="no_toc" id="roadmap">
  
  
    <a href="#roadmap" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Roadmap
  
  
</h1>
    

<div class="toc-block">
<h2 class="no_toc text-delta" id="table-of-contents">
  
  
    <a href="#table-of-contents" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Table of contents
  
  
</h2>
    

<ol id="markdown-toc">
  <li><a href="#ecosystem" id="markdown-toc-ecosystem">Ecosystem</a>    <ol>
      <li><a href="#presigned-url-support-in-lakefshadoopfs-high-priority" id="markdown-toc-presigned-url-support-in-lakefshadoopfs-high-priority">Presigned URL support in LakeFSHadoopFS <span class="label label-blue">High Priority</span></a></li>
      <li><a href="#table-format-support" id="markdown-toc-table-format-support">Table format support</a>        <ol>
          <li><a href="#iceberg-support-high-priority" id="markdown-toc-iceberg-support-high-priority">Iceberg support <span class="label label-blue">High Priority</span></a></li>
          <li><a href="#metastore-integration-hooks" id="markdown-toc-metastore-integration-hooks">Metastore integration hooks</a></li>
          <li><a href="#delta-lake-merges" id="markdown-toc-delta-lake-merges">Delta Lake merges</a></li>
        </ol>
      </li>
      <li><a href="#improved-azure-support" id="markdown-toc-improved-azure-support">Improved Azure Support</a>        <ol>
          <li><a href="#hadoopfs-support-azure-blob-storage" id="markdown-toc-hadoopfs-support-azure-blob-storage">HadoopFS: Support Azure Blob Storage</a></li>
          <li><a href="#support-azure-cosmosdb-as-backend-kv-store" id="markdown-toc-support-azure-cosmosdb-as-backend-kv-store">Support Azure CosmosDB as backend KV store</a></li>
        </ol>
      </li>
      <li><a href="#improved-streaming-support-for-apache-kafka" id="markdown-toc-improved-streaming-support-for-apache-kafka">Improved streaming support for Apache Kafka</a></li>
      <li><a href="#native-connector-trino" id="markdown-toc-native-connector-trino">Native connector: Trino</a></li>
    </ol>
  </li>
  <li><a href="#versioning-capabilities" id="markdown-toc-versioning-capabilities">Versioning Capabilities</a>    <ol>
      <li><a href="#support-long-running-quality-checks" id="markdown-toc-support-long-running-quality-checks">Support long-running quality checks</a></li>
      <li><a href="#git-integration" id="markdown-toc-git-integration">Git Integration</a></li>
    </ol>
  </li>
</ol>

</div><hr />
<h2 id="ecosystem">
  
  
    <a href="#ecosystem" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Ecosystem
  
  
</h2>
    
<h3 id="presigned-url-support-in-lakefshadoopfs-high-priority">
  
  
    <a href="#presigned-url-support-in-lakefshadoopfs-high-priority" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Presigned URL support in LakeFSHadoopFS <span class="label label-blue">High Priority</span>
  
  
</h3>
    

<p>We plan on implementing support for the lakeFS pre-signed URL API into the <a href="/v0.97/integrations/spark.html#use-the-lakefs-hadoop-filesystem">lakeFS Hadoop Filesystem</a>.
Currently, users have to choose between:</p>
<ul>
  <li>managing authorization in lakeFS as well as the object store to allow direct access to lakeFS-managed data</li>
  <li>Use a gateway or proxy component such as the <a href="/v0.97/integrations/spark.html#use-the-s3-compatible-api">S3-compatible API</a> that requires additional sizing</li>
</ul>

<p>Using pre-signed URLs, users will be able to enjoy both worlds: direct access without a proxy, with access control managed in one place</p>

<p>Here’s a simplified example of what this would look like:</p>

<pre><code class="language-mermaid">sequenceDiagram
    autonumber
    Spark Job-&gt;&gt;lakeFS HadoopFS: spark.read.parquet("lakefs://repo/branch/path/")
    lakeFS HadoopFS-&gt;&gt;lakeFS Server: ListObjects("lakefs://repo/branch/path/", presign=True)
    lakeFS Server--&gt;lakeFS Server: Authorize user is allowed to access underlying data
    lakeFS Server--&gt;&gt;lakeFS HadoopFS: ["https://&lt;s3 url&gt;/?X-AMZ-Signature=&lt;sig&gt;", "https://..."]
    par lakeFS HadoopFS to S3
        lakeFS HadoopFS-&gt;&gt;S3: GET http://&lt;s3 url&gt;/?X-AMZ-Signature=&lt;sig&gt;
        S3--&gt;&gt;lakeFS HadoopFS: object content
    end
    lakeFS HadoopFS-&gt;&gt;Spark Job: Parquet Data
</code></pre>

<p><a href="https://github.com/treeverse/lakeFS/pull/5346" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h3 id="table-format-support">
  
  
    <a href="#table-format-support" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Table format support
  
  
</h3>
    
<h4 id="iceberg-support-high-priority">
  
  
    <a href="#iceberg-support-high-priority" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Iceberg support <span class="label label-blue">High Priority</span>
  
  
</h4>
    

<p>A table in Iceberg points to a single metadata file, containing a “location” property. Iceberg uses this location to store:</p>
<ol>
  <li>Manifests describing where the data is stored.</li>
  <li>The actual data.</li>
</ol>

<p>Once a table is created, the location property doesn’t change. Therefore, a branch creation in lakeFS in meaningless, since new data added to this branch will be added to the main branch. There are some workarounds for this, but it is our priority to create an excellent integration with Iceberg.</p>

<p><a href="https://github.com/treeverse/lakeFS/issues/3381" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h4 id="metastore-integration-hooks">
  
  
    <a href="#metastore-integration-hooks" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Metastore integration hooks
  
  
</h4>
    

<p>Allow lakeFS users that query and manipulate data using Hive Metastore to automate the integration between both components.</p>

<ol>
  <li>Creating a branch/tag with a predefined naming convention will result in automatic creation of metastore tables (for a predefined set of table names)</li>
  <li>Deleting a branch/tag with a predefined naming convention will result in automatic deletion of metastore tables (for a predefined set of table names)</li>
  <li>Merging a branch into another branch will result in metastore changes between branches being applied to the metastore for modified tables defined on the destination branch</li>
</ol>

<p><a href="https://github.com/treeverse/lakeFS/issues/3069" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h4 id="delta-lake-merges">
  
  
    <a href="#delta-lake-merges" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Delta Lake merges
  
  
</h4>
    

<p>Delta lake stores metadata files that represent a <a href="https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries">logical transaction log that relies on numerical ordering</a>.</p>

<p>Currently, when trying to modify a Delta table from two different branches, lakeFS would correctly recognize a conflict: this log diverged into two different copies, representing different changes.
Users would then have to forgo one of the change sets by either retaining the destination’s branch set of changes or the source’s branch.</p>

<p>A much better user experience would be to allow merging this log into a new unified set of changes, representing changes made in both branches as a new set of log files (and potentially, data files too!).</p>

<p><a href="https://github.com/treeverse/lakeFS/issues/3380" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h3 id="improved-azure-support">
  
  
    <a href="#improved-azure-support" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Improved Azure Support
  
  
</h3>
    
<h4 id="hadoopfs-support-azure-blob-storage">
  
  
    <a href="#hadoopfs-support-azure-blob-storage" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> HadoopFS: Support Azure Blob Storage
  
  
</h4>
    

<p>Extend the lakeFS HadoopFilesystem to support working directly with Azure Blob Storage (see also: <a href="#presigned-url-support-in-lakefshadoopfs-span-high-priority-span---label-label-blue-">support for pre-signed URLs in the lakeFS Hadoop Filesystem</a> above)</p>

<p><a href="https://github.com/treeverse/lakeFS/issues/5105" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h4 id="support-azure-cosmosdb-as-backend-kv-store">
  
  
    <a href="#support-azure-cosmosdb-as-backend-kv-store" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Support Azure CosmosDB as backend KV store
  
  
</h4>
    

<p>Allow Azure users to use a serverless database such as CosmosDB as their backing KV for lakeFS metadata.</p>

<p>This will also allow auto-provisioning of a CosmosDB database and table as part of the Helm installation process, similar to how its done for DynamoDB on AWS.</p>

<p><a href="https://github.com/treeverse/lakeFS/issues/4458" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h3 id="improved-streaming-support-for-apache-kafka">
  
  
    <a href="#improved-streaming-support-for-apache-kafka" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Improved streaming support for Apache Kafka
  
  
</h3>
    

<p>Committing (along with attaching useful information to the commit) makes a lot of sense for batch workloads:</p>
<ul>
  <li>run a job or a pipeline on a separate branch and commit,</li>
  <li>record information such as the git hash of the code executed, the versions of frameworks used, and information about the data artifacts,</li>
  <li>once the pipeline has completed successfully, commit, and attach the recorded information as metadata.</li>
</ul>

<p>For streaming, however, this is currently less clear: There’s no obvious point in time to commit as things never actually “finish successfully”.
<a href="/v0.97/understand/data_lifecycle_management/production.html#example-1-rollback---data-ingested-from-a-kafka-stream">The recommended pattern</a> would be to ingest from a stream on a separate branch, periodically committing - storing not only the data added since last commit but also capturing the offset read from the stream, for reproducibility.
These commits can then be merged into a main branch given they pass all relevant quality checks and other validations using hooks, exposing consumers to validated, clean data.</p>

<p>In practice, implementing such a workflow is a little challenging. Users need to:</p>

<ol>
  <li>Orchestrate the commits and merge operations.</li>
  <li>Figure out how to attach the correct offset read from the stream broker.</li>
  <li>Handle writes coming in while the commit is taking place.</li>
</ol>

<p>Ideally, lakeFS should provide tools to automate this, with native support for <a href="https://kafka.apache.org/" target="_blank">Apache Kafka</a>.</p>

<p><a href="https://github.com/treeverse/lakeFS/issues/2358" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h3 id="native-connector-trino">
  
  
    <a href="#native-connector-trino" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Native connector: Trino
  
  
</h3>
    

<p>Currently, the Trino integration works well using the <a href="/v0.97/understand/architecture.html#s3-gateway">lakeFS S3 Gateway</a>.</p>

<p>While easy to integrate and useful out-of-the-box, due to the S3 protocol, it means that the data itself must pass through the lakeFS server.</p>

<p>For larger installations, a native integration where lakeFS handles metadata and returns locations in the underlying object store that Trino can then access directly would allow reducing the operational overhead and increasing the scalability of lakeFS.
This would be done in a similar way to the <a href="/v0.97/integrations/spark.html">Native Spark integration</a> using the <a href="/v0.97/integrations/spark.html#use-the-lakefs-hadoop-filesystem">Hadoop Filesystem implementation</a>.</p>

<p><a href="https://github.com/treeverse/lakeFS/issues/2357" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h2 id="versioning-capabilities">
  
  
    <a href="#versioning-capabilities" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Versioning Capabilities
  
  
</h2>
    
<h3 id="support-long-running-quality-checks">
  
  
    <a href="#support-long-running-quality-checks" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Support long-running quality checks
  
  
</h3>
    

<p>Support running quality checks on a branch that might take many minutes to complete.</p>

<p>Currently, <code class="language-plaintext highlighter-rouge">pre-commit</code> and <code class="language-plaintext highlighter-rouge">pre-merge</code> hooks in lakeFS are tied to the lifecycle of the API request that triggers the said commit or merge operation.
In order to support long-running checks, there are enhancements to make to lakeFS APIs in order to support an asynchronous commit and merge operations that are no longer tied to the HTTP request that triggered them.</p>

<p><a href="https://github.com/treeverse/lakeFS/pull/5152" target="_blank" class="btn">Track and discuss it on GitHub</a></p>
<h3 id="git-integration">
  
  
    <a href="#git-integration" class="anchor-heading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Git Integration
  
  
</h3>
    

<p>Support an integration between a lakeFS repository and a Git repository. 
Allow versioning data assets along with the code that was used to modify/generate them.</p>

<p><a href="https://github.com/treeverse/lakeFS/issues/2073" target="_blank" class="btn">Track and discuss it on GitHub</a></p>

                
            </div>
        </div>

        
        <div class="search-overlay"></div>
        
    </div>
</div>

<footer>
    <div class="footer-sidebar"></div>
    <div class="footer-main">
        <div class="row">
            <div class="left">
                <ul class="footer-links">
                    
                    
                    <li class="aux-nav-list-item">
                        <a href="https://docs.lakefs.io/" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            Docs
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/blog/" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            Blog
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://github.com/treeverse/lakeFS" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            GitHub
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/community/" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            Community
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/cloud-registration/" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            lakeFS Cloud
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/contact-us/" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            Contact
                        </a>
                    </li>
                    
                    
                </ul>
            </div>
            <div class="right">
                <ul class="footer-social">
                    
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/youtube" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            <img src="/v0.97/assets/icons/youtube.svg" class="no-hover" />
                            <img src="/v0.97/assets/icons/youtube-hover.svg" class="hover" />
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://www.linkedin.com/company/treeverse" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            <img src="/v0.97/assets/icons/linkedin.svg" class="no-hover" />
                            <img src="/v0.97/assets/icons/linkedin-hover.svg" class="hover" />
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://github.com/treeverse/lakeFS" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            <img src="/v0.97/assets/icons/github.svg" class="no-hover" />
                            <img src="/v0.97/assets/icons/github-hover.svg" class="hover" />
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://twitter.com/lakeFS" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            <img src="/v0.97/assets/icons/twitter.svg" class="no-hover" />
                            <img src="/v0.97/assets/icons/twitter-hover.svg" class="hover" />
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://data-folks.masto.host/@lakeFS" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            <img src="/v0.97/assets/icons/mastodon.svg" class="no-hover" />
                            <img src="/v0.97/assets/icons/mastodon-hover.svg" class="hover" />
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/slack" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            <img src="/v0.97/assets/icons/slack.svg" class="no-hover" />
                            <img src="/v0.97/assets/icons/slack-hover.svg" class="hover" />
                        </a>
                    </li>
                    
                    
                </ul>
            </div>
        </div>
        <div class="row top-border">
            <div class="left">
                <a href="https://lakefs.io" class="lh-tight" style="margin-bottom: 0px;">
                    <div class="site-logo"></div>
                </a>                    
            </div>
            <div class="right">
                <ul class="bottom-footer-links">
                    
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/terms-of-use/" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            Terms of use
                        </a>
                    </li>
                    
                    <li class="aux-nav-list-item">
                        <a href="https://lakefs.io/privacy-policy/" class="site-button"  target="_blank"
                            rel="noopener noreferrer" >
                            Privacy Policy
                        </a>
                    </li>
                    
                    
                </ul>
            </div>
        </div>
    </div>
</footer>


</body>

</html>