Question

我正在尝试连接c＃。

以下是将hive查询成功提交到远程HDInsight群集的类。我需要在这里更改以连接到本地模拟器

public class HadoopImporter : IImporter
{
    public static readonly Logger log = LogManager.GetCurrentClassLogger();

    public void Import(string _query)
    {
        try
        {
            log.Warn("Inside Hive submission method");
            var store = new X509Store();
            store.Open(OpenFlags.ReadOnly);
            var cert =
                store.Certificates.Cast<X509Certificate2>()
                     .First(item => item.Thumbprint == "MYCERTTUMBPRINT");
            if (cert == null)
                log.Error("no cert found");
            log.Warn(cert.FriendlyName);
            log.Warn("got the cert with thumbprint ", cert.Thumbprint.ToString())
            ;

            log.Warn("trying to create credentials from cert");
            var creds = new JobSubmissionCertificateCredential(new Guid("MYSUBSCRIPTIONID"),
                                                               cert, "MYSTORAGECONTAINER");
            log.Warn("trying to connect with cert");
            var jobClient = JobSubmissionClientFactory.Connect(creds);


            log.Warn("Setting Hive job parameters");
            var hiveJob = new HiveJobCreateParameters()
                {
                    Query = _query,
                    StatusFolder = "/samplequeryoutput"
                };

            var jobResults = jobClient.CreateHiveJob(hiveJob);

            log.Warn("Executing wait for jhive results");
            WaitForJobCompletion(jobResults, jobClient);

            using (var stream = jobClient.GetJobOutput(jobResults.JobId))
            {
                var reader = new StreamReader(stream);
                var res = reader.ReadToEnd();
                log.Warn("trying to get the job results " + res.ToString());
            }
        }
        catch (Exception exp)
        {   
            log.Error(exp);

        }
    }


    private static void WaitForJobCompletion(JobCreationResults jobDetails, IJobSubmissionClient client)
    {
        var jobInProgress = client.GetJob(jobDetails.JobId);
        while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
        {
            log.Warn("Inside the while loop waiting for hive job to complete");
            jobInProgress = client.GetJob(jobInProgress.JobId);
            Thread.Sleep(TimeSpan.FromSeconds(10));
        }
        log.Trace("HIVE Job has  Imported " + jobDetails.JobId);
    }
}

Answer 1

您应该能够使用客户端的REST实现连接到本地一体机。

您正在寻找WebHCatHttpClient界面。下面的代码针对我的本地单一框运行基本查询。

var httpClient = new WebHCatHttpClient(new Uri("http://localhost:50111/"), "username", "password");
string outputDir = "basichivejob";
var task = httpClient.CreateHiveJob(@"select * from iris;", null, null, outputDir, null);
task.Wait();
var response = task.Result;
var output = response.Content.ReadAsAsync<JObject>();
output.Wait();
response.EnsureSuccessStatusCode();

string id = output.Result.GetValue("id").ToString();
httpClient.WaitForJobToCompleteAsync(id).Wait();

有关详细信息，请参阅SDK docs。

连接到HDInsight模拟器

1 个答案: