Microsoft Developer Community Blog

15 MIN READ

Exploring Azure Face API: Facial Landmark Detection and Real-Time Analysis with C#

Microsoft

Feb 26, 2026

Azure Face API is a cloud-based vision service offering advanced facial analysis. This article explains how to use facial landmark detection, head pose estimation, and real-time video processing with C# and OpenCV.

In today’s world, applications that understand and respond to human facial cues are no longer science fiction—they’re becoming a reality in domains like security, driver monitoring, gaming, and AR/VR. With Azure Face API, developers can leverage powerful cloud-based facial recognition and analysis tools without building complex machine learning models from scratch.

In this blog, we’ll explore how to use C# to detect faces, identify key facial landmarks, estimate head pose, track eye and mouth movements, and process real-time video streams. Using OpenCV for visualization, we’ll show how to overlay landmarks, draw bounding boxes, and calculate metrics like Eye Aspect Ratio (EAR) and Mouth Aspect Ratio (MAR)—all in real time.

You'll learn to:

Set up Azure Face API
Detect 27 facial landmarks
Estimate head pose (yaw, pitch, roll)
Calculate eye aspect ratio (EAR) and mouth openness
Draw bounding boxes around features using OpenCV
Process real-time video

Prerequisites

.NET 8 SDK installed
Azure subscription with Face API resource
Visual Studio 2022 or later
Webcam for testing (optional)
Basic understanding of C# and computer vision concepts

Part 1: Azure Face API Setup

1.1 Install Required NuGet Packages

dotnet add package Azure.AI.Vision.Face

dotnet add package OpenCvSharp4

dotnet add package OpenCvSharp4.runtime.win

1.2 Create Azure Face API Resource

Navigate to Azure Portal
Search for "Face" and create a new Face API resource
Choose your pricing tier (Free tier: 20 calls/min, 30K calls/month)
Copy the Endpoint URL and API Key

1.3 Configure in .NET Application

appsettings.json:

{

"Azure": {

"FaceApi": {

"Endpoint": "https://your-resource.cognitiveservices.azure.com/",

"ApiKey": "your-api-key-here"

}

}

}

Initialize Face Client:

using Azure;

using Azure.AI.Vision.Face;

using Microsoft.Extensions.Configuration;

public class FaceAnalysisService

{

private readonly FaceClient _faceClient;

private readonly ILogger<FaceAnalysisService> _logger;

public FaceAnalysisService(ILogger<FaceAnalysisService> logger, IConfiguration configuration)

{

_logger = logger;

string endpoint = configuration["Azure:FaceApi:Endpoint"];

string apiKey = configuration["Azure:FaceApi:ApiKey"];

_faceClient = new FaceClient(new Uri(endpoint), new AzureKeyCredential(apiKey));

_logger.LogInformation("FaceClient initialized with endpoint: {Endpoint}", endpoint);

}

}

Part 2: Understanding Face Detection Models

2.1 Basic Face Detection

public async Task<List<FaceDetectionResult>> DetectFacesAsync(byte[] imageBytes)

{

using var stream = new MemoryStream(imageBytes);

var response = await _faceClient.DetectAsync(

BinaryData.FromStream(stream),

FaceDetectionModel.Detection03,

FaceRecognitionModel.Recognition04,

returnFaceId: false,

returnFaceAttributes: new FaceAttributeType[] { FaceAttributeType.HeadPose },

returnFaceLandmarks: true,

returnRecognitionModel: false

);

_logger.LogInformation("Detected {Count} faces", response.Value.Count);

return response.Value.ToList();

}

Part 3: Facial Landmarks - The 27 Key Points

3.1 Understanding Facial Landmarks

3.2 Accessing Landmarks in Code

public void PrintLandmarks(FaceDetectionResult face)

{

var landmarks = face.FaceLandmarks;

if (landmarks == null)

{

_logger.LogWarning("No landmarks detected");

return;

}

// Eye landmarks

Console.WriteLine($"Left Eye Outer: ({landmarks.EyeLeftOuter.X}, {landmarks.EyeLeftOuter.Y})");

Console.WriteLine($"Left Eye Inner: ({landmarks.EyeLeftInner.X}, {landmarks.EyeLeftInner.Y})");

Console.WriteLine($"Left Eye Top: ({landmarks.EyeLeftTop.X}, {landmarks.EyeLeftTop.Y})");

Console.WriteLine($"Left Eye Bottom: ({landmarks.EyeLeftBottom.X}, {landmarks.EyeLeftBottom.Y})");

// Mouth landmarks

Console.WriteLine($"Upper Lip Top: ({landmarks.UpperLipTop.X}, {landmarks.UpperLipTop.Y})");

Console.WriteLine($"Under Lip Bottom: ({landmarks.UnderLipBottom.X}, {landmarks.UnderLipBottom.Y})");

// Nose landmarks

Console.WriteLine($"Nose Tip: ({landmarks.NoseTip.X}, {landmarks.NoseTip.Y})");

}

3.3 Visualizing All Landmarks

public void DrawAllLandmarks(FaceLandmarks landmarks, Mat frame)

{

void DrawPoint(FaceLandmarkCoordinate point, Scalar color)

{

if (point != null)

{

Cv2.Circle(frame, new Point((int)point.X, (int)point.Y), radius: 3, color: color, thickness: -1);

}

}

// Eyes (Green)

DrawPoint(landmarks.EyeLeftOuter, new Scalar(0, 255, 0));

DrawPoint(landmarks.EyeLeftInner, new Scalar(0, 255, 0));

DrawPoint(landmarks.EyeLeftTop, new Scalar(0, 255, 0));

DrawPoint(landmarks.EyeLeftBottom, new Scalar(0, 255, 0));

DrawPoint(landmarks.EyeRightOuter, new Scalar(0, 255, 0));

DrawPoint(landmarks.EyeRightInner, new Scalar(0, 255, 0));

DrawPoint(landmarks.EyeRightTop, new Scalar(0, 255, 0));

DrawPoint(landmarks.EyeRightBottom, new Scalar(0, 255, 0));

// Eyebrows (Cyan)

DrawPoint(landmarks.EyebrowLeftOuter, new Scalar(255, 255, 0));

DrawPoint(landmarks.EyebrowLeftInner, new Scalar(255, 255, 0));

DrawPoint(landmarks.EyebrowRightOuter, new Scalar(255, 255, 0));

DrawPoint(landmarks.EyebrowRightInner, new Scalar(255, 255, 0));

// Nose (Yellow)

DrawPoint(landmarks.NoseTip, new Scalar(0, 255, 255));

DrawPoint(landmarks.NoseRootLeft, new Scalar(0, 255, 255));

DrawPoint(landmarks.NoseRootRight, new Scalar(0, 255, 255));

DrawPoint(landmarks.NoseLeftAlarOutTip, new Scalar(0, 255, 255));

DrawPoint(landmarks.NoseRightAlarOutTip, new Scalar(0, 255, 255));

// Mouth (Blue)

DrawPoint(landmarks.UpperLipTop, new Scalar(255, 0, 0));

DrawPoint(landmarks.UpperLipBottom, new Scalar(255, 0, 0));

DrawPoint(landmarks.UnderLipTop, new Scalar(255, 0, 0));

DrawPoint(landmarks.UnderLipBottom, new Scalar(255, 0, 0));

DrawPoint(landmarks.MouthLeft, new Scalar(255, 0, 0));

DrawPoint(landmarks.MouthRight, new Scalar(255, 0, 0));

// Pupils (Red)

DrawPoint(landmarks.PupilLeft, new Scalar(0, 0, 255));

DrawPoint(landmarks.PupilRight, new Scalar(0, 0, 255));

}

Part 4: Drawing Bounding Boxes Around Features

4.1 Eye Bounding Boxes

/// <summary>

/// Draws rectangles around eyes using OpenCV.

/// </summary>

public void DrawEyeBoxes(FaceLandmarks landmarks, Mat frame)

{

int boxWidth = 60;

int boxHeight = 35;

// Calculate Rectangles

var leftEyeRect = new Rect((int)landmarks.EyeLeftOuter.X - boxWidth / 2, (int)landmarks.EyeLeftOuter.Y - boxHeight / 2, boxWidth, boxHeight);

var rightEyeRect = new Rect((int)landmarks.EyeRightOuter.X - boxWidth / 2, (int)landmarks.EyeRightOuter.Y - boxHeight / 2, boxWidth, boxHeight);

// Draw Rectangles (Green in BGR)

Cv2.Rectangle(frame, leftEyeRect, new Scalar(0, 255, 0), 2);

Cv2.Rectangle(frame, rightEyeRect, new Scalar(0, 255, 0), 2);

// Add Labels

Cv2.PutText(frame, "Left Eye", new Point(leftEyeRect.X, leftEyeRect.Y - 5), HersheyFonts.HersheySimplex, 0.4, new Scalar(0, 255, 0), 1);

Cv2.PutText(frame, "Right Eye", new Point(rightEyeRect.X, rightEyeRect.Y - 5), HersheyFonts.HersheySimplex, 0.4, new Scalar(0, 255, 0), 1);

}

4.2 Mouth Bounding Box

/// <summary>

/// Draws rectangle around mouth region.

/// </summary>

public void DrawMouthBox(FaceLandmarks landmarks, Mat frame)

{

int boxWidth = 80;

int boxHeight = 50;

// Calculate center based on the vertical lip landmarks

int centerX = (int)((landmarks.UpperLipTop.X + landmarks.UnderLipBottom.X) / 2);

int centerY = (int)((landmarks.UpperLipTop.Y + landmarks.UnderLipBottom.Y) / 2);

var mouthRect = new Rect(centerX - boxWidth / 2, centerY - boxHeight / 2, boxWidth, boxHeight);

// Draw Mouth Box (Blue in BGR)

Cv2.Rectangle(frame, mouthRect, new Scalar(255, 0, 0), 2);

// Add Label

Cv2.PutText(frame, "Mouth", new Point(mouthRect.X, mouthRect.Y - 5), HersheyFonts.HersheySimplex, 0.4, new Scalar(255, 0, 0), 1);

}

4.3 Face Bounding Box

/// <summary>

/// Draws rectangle around entire face using the face rectangle from API.

/// </summary>

public void DrawFaceBox(FaceDetectionResult face, Mat frame)

{

var faceRect = face.FaceRectangle;

if (faceRect == null)

{

return;

}

var rect = new Rect(

faceRect.Left,

faceRect.Top,

faceRect.Width,

faceRect.Height

);

// Draw Face Bounding Box (Red in BGR)

Cv2.Rectangle(frame, rect, new Scalar(0, 0, 255), 2);

// Add Label with dimensions

Cv2.PutText(frame, $"Face {faceRect.Width}x{faceRect.Height}", new Point(rect.X, rect.Y - 10), HersheyFonts.HersheySimplex, 0.5,

new Scalar(0, 0, 255), 2);

}

4.4 Nose Bounding Box

/// <summary>

/// Draws bounding box around nose using nose landmarks.

/// </summary>

public void DrawNoseBox(FaceLandmarks landmarks, Mat frame)

{

// Calculate horizontal bounds from Alar tips

int minX = (int)Math.Min(landmarks.NoseLeftAlarOutTip.X, landmarks.NoseRightAlarOutTip.X);

int maxX = (int)Math.Max(landmarks.NoseLeftAlarOutTip.X, landmarks.NoseRightAlarOutTip.X);

// Calculate vertical bounds from Root to Tip

int minY = (int)Math.Min(landmarks.NoseRootLeft.Y, landmarks.NoseTip.Y);

int maxY = (int)landmarks.NoseTip.Y;

// Create Rect with a 10px padding buffer

var noseRect = new Rect(

minX - 10,

minY - 10,

(maxX - minX) + 20,

(maxY - minY) + 20

);

// Draw Nose Box (Yellow in BGR)

Cv2.Rectangle(frame, noseRect, new Scalar(0, 255, 255), 2);

}

Part 5: Geometric Calculations with Landmarks

5.1 Calculating Euclidean Distance

/// <summary>

/// Calculates distance between two landmark points.

/// </summary>

public static double CalculateDistance(dynamic point1, dynamic point2)

{

double dx = point1.X - point2.X;

double dy = point1.Y - point2.Y;

return Math.Sqrt(dx * dx + dy * dy);

}

5.2 Eye Aspect Ratio (EAR) Formula

/// <summary>

/// Calculates the Eye Aspect Ratio (EAR) to detect eye closure.

/// </summary>

public double CalculateEAR(

FaceLandmarkCoordinate top1,

FaceLandmarkCoordinate top2,

FaceLandmarkCoordinate bottom1,

FaceLandmarkCoordinate bottom2,

FaceLandmarkCoordinate inner,

FaceLandmarkCoordinate outer)

{

// Vertical distances

double v1 = CalculateDistance(top1, bottom1);

double v2 = CalculateDistance(top2, bottom2);

// Horizontal distance

double h = CalculateDistance(inner, outer);

// EAR formula: (||p2-p6|| + ||p3-p5||) / (2 * ||p1-p4||)

return (v1 + v2) / (2.0 * h);

}

Simplified Implementation:

/// <summary>

/// Calculates Eye Aspect Ratio (EAR) for a single eye.

/// Reference: "Real-Time Eye Blink Detection using Facial Landmarks" (Soukupová & Čech, 2016)

/// </summary>

public double ComputeEAR(FaceLandmarks landmarks, bool isLeftEye)

{

var top = isLeftEye ? landmarks.EyeLeftTop : landmarks.EyeRightTop;

var bottom = isLeftEye ? landmarks.EyeLeftBottom : landmarks.EyeRightBottom;

var inner = isLeftEye ? landmarks.EyeLeftInner : landmarks.EyeRightInner;

var outer = isLeftEye ? landmarks.EyeLeftOuter : landmarks.EyeRightOuter;

if (top == null || bottom == null || inner == null || outer == null)

{

_logger.LogWarning("Missing eye landmarks");

return 1.0; // Return 1.0 (open) to prevent false positives for drowsiness

}

double verticalDist = CalculateDistance(top, bottom);

double horizontalDist = CalculateDistance(inner, outer);

// Simplified EAR for Azure 27-point model

double ear = verticalDist / horizontalDist;

_logger.LogDebug(

"EAR for {Eye}: {Value:F3}",

isLeftEye ? "left" : "right",

ear

);

return ear;

}

Usage Example:

var leftEAR = ComputeEAR(landmarks, isLeftEye: true);

var rightEAR = ComputeEAR(landmarks, isLeftEye: false);

var avgEAR = (leftEAR + rightEAR) / 2.0;

Console.WriteLine($"Average EAR: {avgEAR:F3}");

// Open eyes: ~0.25-0.30

// Closed eyes: ~0.10-0.15

5.3 Mouth Aspect Ratio (MAR)

/// <summary>

/// Calculates Mouth Aspect Ratio relative to face height.

/// </summary>

public double CalculateMouthAspectRatio(FaceLandmarks landmarks, FaceRectangle faceRect)

{

double mouthHeight = landmarks.UnderLipBottom.Y - landmarks.UpperLipTop.Y;

double mouthWidth = CalculateDistance(landmarks.MouthLeft, landmarks.MouthRight);

double mouthOpenRatio = mouthHeight / faceRect.Height;

double mouthWidthRatio = mouthWidth / faceRect.Width;

_logger.LogDebug(

"Mouth - Height ratio: {HeightRatio:F3}, Width ratio: {WidthRatio:F3}",

mouthOpenRatio,

mouthWidthRatio

);

return mouthOpenRatio;

}

5.4 Inter-Eye Distance

/// <summary>

/// Calculates the distance between pupils (inter-pupillary distance).

/// </summary>

public double CalculateInterEyeDistance(FaceLandmarks landmarks)

{

return CalculateDistance(landmarks.PupilLeft, landmarks.PupilRight);

}

/// <summary>

/// Calculates distance between inner eye corners.

/// </summary>

public double CalculateInnerEyeDistance(FaceLandmarks landmarks)

{

return CalculateDistance(landmarks.EyeLeftInner, landmarks.EyeRightInner);

}

5.5 Face Symmetry Analysis

/// <summary>

/// Analyzes facial symmetry by comparing left and right sides.

/// </summary>

public FaceSymmetryMetrics AnalyzeFaceSymmetry(FaceLandmarks landmarks)

{

double centerX = landmarks.NoseTip.X;

double leftEyeDistance = CalculateDistance(landmarks.EyeLeftInner, new { X = centerX, Y = landmarks.EyeLeftInner.Y });

double leftMouthDistance = CalculateDistance(landmarks.MouthLeft, new { X = centerX, Y = landmarks.MouthLeft.Y });

double rightEyeDistance = CalculateDistance(landmarks.EyeRightInner, new { X = centerX, Y = landmarks.EyeRightInner.Y });

double rightMouthDistance = CalculateDistance(landmarks.MouthRight, new { X = centerX, Y = landmarks.MouthRight.Y });

return new FaceSymmetryMetrics

{

EyeSymmetryRatio = leftEyeDistance / rightEyeDistance,

MouthSymmetryRatio = leftMouthDistance / rightMouthDistance,

IsSymmetric = Math.Abs(leftEyeDistance - rightEyeDistance) < 5.0

};

}

public class FaceSymmetryMetrics

{

public double EyeSymmetryRatio { get; set; }

public double MouthSymmetryRatio { get; set; }

public bool IsSymmetric { get; set; }

}

Part 6: Head Pose Estimation

6.1 Understanding Head Pose Angles

Azure Face API provides three Euler angles for head orientation:

6.2 Accessing Head Pose Data

public void AnalyzeHeadPose(FaceDetectionResult face)

{

var headPose = face.FaceAttributes?.HeadPose;

if (headPose == null)

{

_logger.LogWarning("Head pose not available");

return;

}

double yaw = headPose.Yaw;

double pitch = headPose.Pitch;

double roll = headPose.Roll;

Console.WriteLine("Head Pose:");

Console.WriteLine($" Yaw: {yaw:F2}° (Left/Right)");

Console.WriteLine($" Pitch: {pitch:F2}° (Up/Down)");

Console.WriteLine($" Roll: {roll:F2}° (Tilt)");

InterpretHeadPose(yaw, pitch, roll);

}

6.3 Interpreting Head Pose

public string InterpretHeadPose(double yaw, double pitch, double roll) {

var directions = new List<string>();

// Interpret Yaw (horizontal)

if (Math.Abs(yaw) < 10) directions.Add("Looking Forward");

else if (yaw < -20) directions.Add($"Turned Left ({Math.Abs(yaw):F0}°)");

else if (yaw > 20) directions.Add($"Turned Right ({yaw:F0}°)");

// Interpret Pitch (vertical)

if (Math.Abs(pitch) < 10) directions.Add("Level");

else if (pitch < -15) directions.Add($"Looking Down ({Math.Abs(pitch):F0}°)");

else if (pitch > 15) directions.Add($"Looking Up ({pitch:F0}°)");

// Interpret Roll (tilt)

if (Math.Abs(roll) > 15) {

string side = roll < 0 ? "Left" : "Right";

directions.Add($"Tilted {side} ({Math.Abs(roll):F0}°)");

}

return string.Join(", ", directions);

}

6.4 Visualizing Head Pose on Frame

/// <summary>

/// Draws head pose information with color-coded indicators.

/// </summary>

public void DrawHeadPoseInfo(Mat frame, HeadPose headPose, FaceRectangle faceRect)

{

double yaw = headPose.Yaw;

double pitch = headPose.Pitch;

double roll = headPose.Roll;

int centerX = faceRect.Left + faceRect.Width / 2;

int centerY = faceRect.Top + faceRect.Height / 2;

string poseText = $"Yaw: {yaw:F1}° Pitch: {pitch:F1}° Roll: {roll:F1}°";

Cv2.PutText(frame, poseText, new Point(faceRect.Left, faceRect.Top - 10), HersheyFonts.HersheySimplex, 0.5, new Scalar(255, 255, 255), 1);

int arrowLength = 50;

double yawRadians = yaw * Math.PI / 180.0;

int arrowEndX = centerX + (int)(arrowLength * Math.Sin(yawRadians));

Cv2.ArrowedLine(frame, new Point(centerX, centerY), new Point(arrowEndX, centerY), new Scalar(0, 255, 0), 2, tipLength: 0.3);

double pitchRadians = -pitch * Math.PI / 180.0;

int arrowPitchEndY = centerY + (int)(arrowLength * Math.Sin(pitchRadians));

Cv2.ArrowedLine(frame, new Point(centerX, centerY), new Point(centerX, arrowPitchEndY), new Scalar(255, 0, 0), 2, tipLength: 0.3);

}

6.5 Detecting Head Orientation States

public enum HeadOrientation { Forward, Left, Right, Up, Down, TiltedLeft, TiltedRight, UpLeft, UpRight, DownLeft, DownRight }

public List<HeadOrientation> DetectHeadOrientation(HeadPose headPose)

{

const double THRESHOLD = 15.0;

bool lookingUp = headPose.Pitch > THRESHOLD;

bool lookingDown = headPose.Pitch < -THRESHOLD;

bool lookingLeft = headPose.Yaw < -THRESHOLD;

bool lookingRight = headPose.Yaw > THRESHOLD;

var orientations = new List<HeadOrientation>();

if (!lookingUp && !lookingDown && !lookingLeft && !lookingRight) orientations.Add(HeadOrientation.Forward);

if (lookingUp && !lookingLeft && !lookingRight) orientations.Add(HeadOrientation.Up);

if (lookingDown && !lookingLeft && !lookingRight) orientations.Add(HeadOrientation.Down);

if (lookingLeft && !lookingUp && !lookingDown) orientations.Add(HeadOrientation.Left);

if (lookingRight && !lookingUp && !lookingDown) orientations.Add(HeadOrientation.Right);

if (lookingUp && lookingLeft) orientations.Add(HeadOrientation.UpLeft);

if (lookingUp && lookingRight) orientations.Add(HeadOrientation.UpRight);

if (lookingDown && lookingLeft) orientations.Add(HeadOrientation.DownLeft);

if (lookingDown && lookingRight) orientations.Add(HeadOrientation.DownRight);

return orientations;

}

Part 7: Real-Time Video Processing

7.1 Setting Up Video Capture

using OpenCvSharp;

public class RealTimeFaceAnalyzer : IDisposable

{

private VideoCapture? _capture;

private Mat? _frame;

private readonly FaceClient _faceClient;

private bool _isRunning;

public async Task StartAsync()

{

_capture = new VideoCapture(0);

_frame = new Mat();

_isRunning = true;

await Task.Run(() => ProcessVideoLoop());

}

private async Task ProcessVideoLoop()

{

while (_isRunning)

{

if (_capture == null || !_capture.IsOpened()) break;

_capture.Read(_frame);

if (_frame == null || _frame.Empty())

{

await Task.Delay(1); // Minimal delay to prevent CPU spiking

continue;

}

Cv2.Resize(_frame, _frame, new Size(640, 480));

// Ensure we don't await indefinitely in the rendering loop

_ = ProcessFrameAsync(_frame.Clone());

Cv2.ImShow("Face Analysis", _frame);

if (Cv2.WaitKey(30) == 'q') break;

}

Dispose();

}

private async Task ProcessFrameAsync(Mat frame)

{

// This is where your DrawFaceBox, DrawAllLandmarks, and EAR logic will sit.

// Remember to use try-catch here to prevent API errors from crashing the loop.

}

public void Dispose()

{

_isRunning = false;

_capture?.Dispose();

_frame?.Dispose();

Cv2.DestroyAllWindows();

}

}

7.2 Optimizing API Calls

Problem: Calling Azure Face API on every frame (30 fps) is expensive and slow.

Solution: Call API once per second, cache results for 30 frames.

private List<FaceDetectionResult> _cachedFaces = new();

private DateTime _lastDetectionTime = DateTime.MinValue;

private readonly object _cacheLock = new();

private async Task ProcessFrameAsync(Mat frame)

{

if ((DateTime.Now - _lastDetectionTime).TotalSeconds >= 1.0)

{

_lastDetectionTime = DateTime.Now;

byte[] imageBytes;

Cv2.ImEncode(".jpg", frame, out imageBytes);

var faces = await DetectFacesAsync(imageBytes);

lock (_cacheLock)

{

_cachedFaces = faces;

}

}

List<FaceDetectionResult> facesToProcess;

lock (_cacheLock)

{

facesToProcess = _cachedFaces.ToList();

}

foreach (var face in facesToProcess)

{

DrawFaceAnnotations(face, frame);

}

}

Performance Improvement:

30x fewer API calls (1/sec instead of 30/sec)
~$0.02/hour instead of ~$0.60/hour
Smooth 30 fps rendering
< 100ms latency for visual updates

7.3 Drawing Complete Face Annotations

private void DrawFaceAnnotations(FaceDetectionResult face, Mat frame)

{

DrawFaceBox(face, frame);

if (face.FaceLandmarks != null)

{

DrawAllLandmarks(face.FaceLandmarks, frame);

DrawEyeBoxes(face.FaceLandmarks, frame);

DrawMouthBox(face.FaceLandmarks, frame);

DrawNoseBox(face.FaceLandmarks, frame);

double leftEAR = ComputeEAR(face.FaceLandmarks, isLeftEye: true);

double rightEAR = ComputeEAR(face.FaceLandmarks, isLeftEye: false);

double avgEAR = (leftEAR + rightEAR) / 2.0;

Cv2.PutText(frame, $"EAR: {avgEAR:F3}", new Point(10, 30), HersheyFonts.HersheySimplex, 0.6, new Scalar(0, 255, 0), 2);

}

if (face.FaceAttributes?.HeadPose != null)

{

DrawHeadPoseInfo(frame, face.FaceAttributes.HeadPose, face.FaceRectangle);

string orientation = InterpretHeadPose(face.FaceAttributes.HeadPose.Yaw, face.FaceAttributes.HeadPose.Pitch, face.FaceAttributes.HeadPose.Roll);

Cv2.PutText(frame, orientation, new Point(10, 60), HersheyFonts.HersheySimplex, 0.6, new Scalar(255, 255, 0), 2);

}

}

Part 8: Advanced Features and Use Cases

8.1 Face Tracking Across Frames

public class FaceTracker

{

private class TrackedFace

{

public FaceRectangle Rectangle { get; set; }

public DateTime LastSeen { get; set; }

public int TrackId { get; set; }

}

private List<TrackedFace> _trackedFaces = new();

private int _nextTrackId = 1;

public int TrackFace(FaceRectangle newFace)

{

const int MATCH_THRESHOLD = 50;

var match = _trackedFaces.FirstOrDefault(tf => {

double distance = Math.Sqrt(Math.Pow(tf.Rectangle.Left - newFace.Left, 2) + Math.Pow(tf.Rectangle.Top - newFace.Top, 2));

return distance < MATCH_THRESHOLD;

});

if (match != null)

{

match.Rectangle = newFace;

match.LastSeen = DateTime.Now;

return match.TrackId;

}

var newTrack = new TrackedFace { Rectangle = newFace, LastSeen = DateTime.Now, TrackId = _nextTrackId++ };

_trackedFaces.Add(newTrack);

return newTrack.TrackId;

}

public void RemoveOldTracks(TimeSpan maxAge)

{

_trackedFaces.RemoveAll(tf => DateTime.Now - tf.LastSeen > maxAge);

}

}

8.2 Multi-Face Detection and Analysis

public async Task<FaceAnalysisReport> AnalyzeMultipleFacesAsync(byte[] imageBytes)

{

var faces = await DetectFacesAsync(imageBytes);

var report = new FaceAnalysisReport { TotalFacesDetected = faces.Count, Timestamp = DateTime.Now, Faces = new List<SingleFaceAnalysis>() };

for (int i = 0; i < faces.Count; i++)

{

var face = faces[i];

var analysis = new SingleFaceAnalysis { FaceIndex = i, FaceLocation = face.FaceRectangle, FaceSize = face.FaceRectangle.Width * face.FaceRectangle.Height };

if (face.FaceLandmarks != null)

{

analysis.LeftEyeEAR = ComputeEAR(face.FaceLandmarks, true);

analysis.RightEyeEAR = ComputeEAR(face.FaceLandmarks, false);

analysis.InterPupillaryDistance = CalculateInterEyeDistance(face.FaceLandmarks);

}

if (face.FaceAttributes?.HeadPose != null)

{

analysis.HeadYaw = face.FaceAttributes.HeadPose.Yaw;

analysis.HeadPitch = face.FaceAttributes.HeadPose.Pitch;

analysis.HeadRoll = face.FaceAttributes.HeadPose.Roll;

}

report.Faces.Add(analysis);

}

report.Faces = report.Faces.OrderByDescending(f => f.FaceSize).ToList();

return report;

}

public class FaceAnalysisReport

{

public int TotalFacesDetected { get; set; }

public DateTime Timestamp { get; set; }

public List<SingleFaceAnalysis> Faces { get; set; }

}

public class SingleFaceAnalysis

{

public int FaceIndex { get; set; }

public FaceRectangle FaceLocation { get; set; }

public int FaceSize { get; set; }

public double LeftEyeEAR { get; set; }

public double RightEyeEAR { get; set; }

public double InterPupillaryDistance { get; set; }

public double HeadYaw { get; set; }

public double HeadPitch { get; set; }

public double HeadRoll { get; set; }

}

8.3 Exporting Landmark Data to JSON

using System.Text.Json;

public string ExportLandmarksToJson(FaceDetectionResult face)

{

var landmarks = face.FaceLandmarks;

var landmarkData = new

{

Face = new { Rectangle = new { face.FaceRectangle.Left, face.FaceRectangle.Top, face.FaceRectangle.Width, face.FaceRectangle.Height } },

Eyes = new

{

Left = new { Outer = new { landmarks.EyeLeftOuter.X, landmarks.EyeLeftOuter.Y }, Inner = new { landmarks.EyeLeftInner.X, landmarks.EyeLeftInner.Y }, Top = new { landmarks.EyeLeftTop.X, landmarks.EyeLeftTop.Y }, Bottom = new { landmarks.EyeLeftBottom.X, landmarks.EyeLeftBottom.Y } },

Right = new { Outer = new { landmarks.EyeRightOuter.X, landmarks.EyeRightOuter.Y }, Inner = new { landmarks.EyeRightInner.X, landmarks.EyeRightInner.Y }, Top = new { landmarks.EyeRightTop.X, landmarks.EyeRightTop.Y }, Bottom = new { landmarks.EyeRightBottom.X, landmarks.EyeRightBottom.Y } }

},

Mouth = new { UpperLipTop = new { landmarks.UpperLipTop.X, landmarks.UpperLipTop.Y }, UnderLipBottom = new { landmarks.UnderLipBottom.X, landmarks.UnderLipBottom.Y }, Left = new { landmarks.MouthLeft.X, landmarks.MouthLeft.Y }, Right = new { landmarks.MouthRight.X, landmarks.MouthRight.Y } },

Nose = new { Tip = new { landmarks.NoseTip.X, landmarks.NoseTip.Y }, RootLeft = new { landmarks.NoseRootLeft.X, landmarks.NoseRootLeft.Y }, RootRight = new { landmarks.NoseRootRight.X, landmarks.NoseRootRight.Y } },

HeadPose = face.FaceAttributes?.HeadPose != null ? new { face.FaceAttributes.HeadPose.Yaw, face.FaceAttributes.HeadPose.Pitch, face.FaceAttributes.HeadPose.Roll } : null

};

return JsonSerializer.Serialize(landmarkData, new JsonSerializerOptions { WriteIndented = true });

}

Part 9: Practical Applications

9.1 Gaze Direction Estimation

public enum GazeDirection { Center, Left, Right, Up, Down, UpLeft, UpRight, DownLeft, DownRight }

public GazeDirection EstimateGazeDirection(HeadPose headPose)

{

const double THRESHOLD = 15.0;

bool lookingUp = headPose.Pitch > THRESHOLD;

bool lookingDown = headPose.Pitch < -THRESHOLD;

bool lookingLeft = headPose.Yaw < -THRESHOLD;

bool lookingRight = headPose.Yaw > THRESHOLD;

if (lookingUp && lookingLeft) return GazeDirection.UpLeft;

if (lookingUp && lookingRight) return GazeDirection.UpRight;

if (lookingDown && lookingLeft) return GazeDirection.DownLeft;

if (lookingDown && lookingRight) return GazeDirection.DownRight;

if (lookingUp) return GazeDirection.Up;

if (lookingDown) return GazeDirection.Down;

if (lookingLeft) return GazeDirection.Left;

if (lookingRight) return GazeDirection.Right;

return GazeDirection.Center;

}

9.2 Expression Analysis Using Landmarks

public class ExpressionAnalyzer

{

public bool IsSmiling(FaceLandmarks landmarks)

{

double mouthCenterY = (landmarks.UpperLipTop.Y + landmarks.UnderLipBottom.Y) / 2;

double leftCornerY = landmarks.MouthLeft.Y;

double rightCornerY = landmarks.MouthRight.Y;

return leftCornerY < mouthCenterY && rightCornerY < mouthCenterY;

}

public bool IsMouthOpen(FaceLandmarks landmarks, FaceRectangle faceRect)

{

double mouthHeight = landmarks.UnderLipBottom.Y - landmarks.UpperLipTop.Y;

double mouthOpenRatio = mouthHeight / faceRect.Height;

return mouthOpenRatio > 0.08; // 8% of face height

}

public bool AreEyesClosed(FaceLandmarks landmarks)

{

double leftEAR = ComputeEAR(landmarks, isLeftEye: true);

double rightEAR = ComputeEAR(landmarks, isLeftEye: false);

double avgEAR = (leftEAR + rightEAR) / 2.0;

return avgEAR < 0.18; // Threshold for closed eyes

}

}

9.3 Face Orientation for AR/VR Applications

public class FaceOrientationFor3D

{

public (Vector3 forward, Vector3 up, Vector3 right) GetFaceOrientation(HeadPose headPose)

{

double yawRad = headPose.Yaw * Math.PI / 180.0;

double pitchRad = headPose.Pitch * Math.PI / 180.0;

double rollRad = headPose.Roll * Math.PI / 180.0;

var forward = new Vector3((float)(Math.Sin(yawRad) * Math.Cos(pitchRad)), (float)(-Math.Sin(pitchRad)), (float)(Math.Cos(yawRad) * Math.Cos(pitchRad)));

var up = new Vector3((float)(Math.Sin(yawRad) * Math.Sin(pitchRad) * Math.Cos(rollRad) - Math.Cos(yawRad) * Math.Sin(rollRad)), (float)(Math.Cos(pitchRad) * Math.Cos(rollRad)), (float)(Math.Cos(yawRad) * Math.Sin(pitchRad) * Math.Cos(rollRad) + Math.Sin(yawRad) * Math.Sin(rollRad)));

var right = Vector3.Cross(up, forward);

return (forward, up, right);

}

}

public struct Vector3

{

public float X, Y, Z;

public Vector3(float x, float y, float z) { X = x; Y = y; Z = z; }

public static Vector3 Cross(Vector3 a, Vector3 b) => new Vector3(a.Y * b.Z - a.Z * b.Y, a.Z * b.X - a.X * b.Z, a.X * b.Y - a.Y * b.X);

}