来自ADLS文件的Databricks作业小部件

# 示例代码：读取ADLS中的数据
from pyspark.sql import SparkSession

spark = SparkSession.builder \
    .appName("Read from ADLS") \
    .config("fs.adl.oauth2.access.token.provider.type", "ClientCredential") \
    .config("fs.adl.oauth2.client.id", "<client_id>") \
    .config("fs.adl.oauth2.credential", "<client_secret>") \
    .config("fs.adl.oauth2.refresh.url", "https://login.microsoftonline.com/<tenant_id>/oauth2/token") \
    .getOrCreate()

df = spark.read.csv("adl://<storage_account>.azuredatalakestore.net/<file_path>")
df.show()

# 示例代码：提交Databricks作业
from databricks_sdk import DatabricksClient

client = DatabricksClient(host="<databricks_host>", token="<databricks_token>")

job_id = client.jobs.create_job(
    name="My Job",
    existing_cluster_id="<cluster_id>",
    notebook_params={"input_path": "/path/to/input"},
    notebook_task={"notebook_path": "/path/to/notebook"}
)

client.jobs.run_now(job_id=job_id)