Database Reference
In-Depth Information
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId,
cert, Constants.clusterName);
var jobClient = JobSubmissionClientFactory.Connect(creds);
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
Console.Write("Executing Hive Job.");
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Print the Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
Console.Write("Done..List of Tables are:\n");
Console.WriteLine(reader.ReadToEnd());
}
//Monitor cluster Map Reduce statistics
public static void MonitorCluster()
{
var client = new AmbariClient(Constants.azureClusterUri,
Constants.clusterUser, Constants.clusterPassword);
IList<ClusterInfo> clusterInfos = client.GetClusters();
ClusterInfo clusterInfo = clusterInfos[0];
Console.WriteLine("Cluster Href: {0}", clusterInfo.Href);
Regex clusterNameRegEx = new Regex(@"(\w+)\.*");
var clusterName =
clusterNameRegEx.Match(Constants.azureClusterUri.Authority).Groups[1].Value;
HostComponentMetric hostComponentMetric = client.GetHostComponentMetric(
clusterName + ".azurehdinsight.net");
Console.WriteLine("Cluster Map Reduce Metrics:");
Console.WriteLine("\tMaps Completed: \t{0}", hostComponentMetric.MapsCompleted);
Console.WriteLine("\tMaps Failed: \t{0}", hostComponentMetric.MapsFailed);
Console.WriteLine("\tMaps Killed: \t{0}", hostComponentMetric.MapsKilled);
Console.WriteLine("\tMaps Launched: \t{0}", hostComponentMetric.MapsLaunched);
Console.WriteLine("\tMaps Running: \t{0}", hostComponentMetric.MapsRunning);
Console.WriteLine("\tMaps Waiting: \t{0}", hostComponentMetric.MapsWaiting);
}
///Helper Function to Wait while job executes
private static void WaitForJobCompletion(JobCreationResults jobResults,
IJobSubmissionClient client)
{
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed &&
jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(1));
Console.Write(".");
}
}
}
}
 
Search WWH ::




Custom Search