Software Design Blog

Processing everything in real-time is easy but often makes an application tightly coupled, prone to failure and difficult to scale.

Let’s assume a user just clicked on the purchase button to kick off the following workflow:

1. Save the order
2. Send a confirmation email (mask the credit card number)
3. Update the user’s loyalty points in an external system

The problem is that non-critical workloads (step 2 and 3) can negatively impact an application's performance and recoverability. What happens when the mail server is down and the email confirmation fails? How do we monitor failure? Do we roll the transaction back or replay the workflow by taking a checkpoint after each successful step?

The solution is to ensure that application flow isn't impeded by waiting on non-critical workloads to complete. Queue based systems are effective at solving this problem.

This post will use a purchase order application that accepts XML requests. The application will sanitise the credit card details by masking most of the digits before sending the request to a mailbox.

Setup

The following XML document will be used as the purchase order. It is about 512KB to simulate a decent payload.

  


  
    Test User
    
123 Abc Road, Sydney, Australia
    
      Visa
      4111111111111111
    
  
  
    
      Gambardella, Matthew
      
    
   ...

  
    public interface IMailer
    {
        void Send(string message);
    }

    public class PurchaseOrderProcessor
    {
        private readonly IMailer _mailer;

        public PurchaseOrderProcessor(IMailer mailer)
        {
            if (mailer == null) throw new ArgumentNullException("mailer");
            _mailer = mailer;
        }

        public void Process(string xmlRequest)
        {
            var doc = new XmlDocument();
            doc.LoadXml(xmlRequest);

            // Process the order

            _mailer.Send(xmlRequest);
        }
    }

  
    public class FileSmtpMailer : IMailer
    {
        private readonly string _path;
        private readonly string _from;
        private readonly string _recipients;
        private readonly string _subject;

        public FileSmtpMailer(string path, string from, string recipients, string subject)
        {
            if (path == null) throw new ArgumentNullException("path");
            _path = path;
            _from = @from;
            _recipients = recipients;
            _subject = subject;
        }

        public void Send(string message)
        {
            using (var client = new SmtpClient())
            {
                // This can be configured in the app.config
                client.DeliveryMethod = SmtpDeliveryMethod.SpecifiedPickupDirectory;
                client.PickupDirectoryLocation = _path;

                using (var mailMessage = 
                          new MailMessage(_from, _recipients, _subject, message))
                {
                    client.IsBodyHtml = true;
                    client.Send(mailMessage);    
                }             
            }
        }
    }

  
    public class MaskedMailerDecorator : IMailer
    {
        private readonly Regex _validationRegex;
        private readonly IMailer _next;
        private const char MaskCharacter = '*';
        private const int MaskDigits = 4;

        public MaskedMailerDecorator(Regex regex, IMailer next)
        {
            if (regex == null) throw new ArgumentNullException("regex");
            if (next == null) throw new ArgumentNullException("next");
            _validationRegex = regex;
            _next = next;
        }

        public void Send(string message)
        {
            if (_validationRegex.IsMatch(message))
            {
                message = _validationRegex.Replace(message, 
                               match => MaskNumber(match.Value));
            }
            _next.Send(message);
        }

        private static string MaskNumber(string value)
        {
            return value.Length <= MaskDigits ?
               new string(MaskCharacter, value.Length) :
               string.Format("{0}{1}", 
                          new string(MaskCharacter, value.Length - MaskDigits),
                          value.Substring(value.Length - MaskDigits, MaskDigits));
        }
    }

Baseline

Let's establish a performance baseline by running the application with the Null Mailer that doesn't do anything. Refer to the reject the null checked object post if you are new to the null object pattern.

  
    public class NullMailer : IMailer
    {
        public void Send(string message)
        {
            // intentionally blank
        }
    }

    static void Main(string[] args)
    {
        var path = Path.Combine(Directory.GetCurrentDirectory(), "PurchaseOrder.xml");
        var request = File.ReadAllText(path); 

        var nullMailer = new NullMailer();
        var orderProcessor = new PurchaseOrderProcessor(nullMailer);

        var stopWatch = Stopwatch.StartNew();
        Parallel.For(0, 1000, i => orderProcessor.Process(request));
        stopWatch.Stop();

        Console.WriteLine("Seconds: {0}", stopWatch.Elapsed.TotalSeconds);
        Console.ReadLine();
    }

Seconds: 6.3404086

Real-Time Processing

  
            Directory.CreateDirectory(@"C:\temp");
            var ccRegEx = new Regex(@"(?:\b4[0-9]{12}(?:[0-9]{3})?\b
                                       |\b5[1-5][0-9]{14}\b)", RegexOptions.Compiled);

            var path = Path.Combine(Directory.GetCurrentDirectory(), "PurchaseOrder.xml");
            var request = File.ReadAllText(path);
            
            // Use Unity for doing the wiring          
            var fileMailer = new FileSmtpMailer(@"C:\temp", "p@mail.com", "a@mail.com", "Order");
            var maskedMailer = new MaskedMailerDecorator(ccRegEx, fileMailer);
            var orderProcessor = new PurchaseOrderProcessor(maskedMailer);

            var stopWatch = Stopwatch.StartNew();
            Parallel.For(0, 1000, i => orderProcessor.Process(request));
            stopWatch.Stop();

            Console.WriteLine("Seconds: {0}", stopWatch.Elapsed.TotalSeconds);
            Console.ReadLine();

Seconds: 32.0430142

Background Processing

Let's extend the solution by adding a memory queue to buffer the results. The queue effectively acts as an outbox for sending mail without overwhelming the mail server with parallel requests.

    public class QueuedMailerDecorator : IMailer
    {
        private readonly IMailer _next;
        private BlockingCollection<string> _messages;

        public QueuedMailerDecorator(IMailer next)
        {
            if (next == null) throw new ArgumentNullException("next");
            _next = next;
            _messages = new BlockingCollection<string>();

            Task.Factory.StartNew(() =>
            {
                try
                {
                    // Block the thread until a message becomes available
                    foreach (var message in _messages.GetConsumingEnumerable())
                    {
                        _next.Send(message);
                    }
                }
                finally
                {
                    _messages.Dispose();
                }
            }, TaskCreationOptions.LongRunning);
        }

        public void Send(string message)
        {
            if (_messages == null || _messages.IsAddingCompleted)
            {
                return;
            }
            try
            {
                _messages.TryAdd(message);
            }
            catch (ObjectDisposedException)
            {
                Trace.WriteLine("Add failed since the queue was disposed.");
            }
        }

        public void Dispose()
        {
            if (_messages != null && !_messages.IsAddingCompleted)
            {
                _messages.CompleteAdding();
            }
            GC.SuppressFinalize(this);
        }
    }

Here is a diagram that depicts the collection of decorated mailers to intercept the communication between the PurchaseOrderProcessor and the FileSmtpMailer.

             
            // Use Unity for doing the wiring          
            var fileMailer = new FileSmtpMailer(@"C:\temp", "p@mail.com", "a@mail.com", "Order");
            var maskedMailer = new MaskedMailerDecorator(creditCardRegEx, fileMailer);
            var queuedMailer = new QueuedMailerDecorator(maskedMailer);
            var orderProcessor = new PurchaseOrderProcessor(queuedMailer);

            var stopWatch = Stopwatch.StartNew();
            Parallel.For(0, 1000, i => orderProcessor.Process(request));
            stopWatch.Stop();

            Console.WriteLine("Seconds: {0}", stopWatch.Elapsed.TotalSeconds);
            Console.ReadLine();

Seconds: 6.3908034 

The drawback of the in-memory queue used in this post is that it requires memory to store messages temporarily before it is processed. It is possible to lose messages if the application crashes or stops unexpectedly before all of the requests have been processed.

These issues can be addressed with a locally persisted queue such as MSMQ or a cloud based queue such as Azure Service Bus. Queues provide many benefits that will be covered in the next post.

Summary

Calling optional dependencies such as logging, tracing and notifications should be fast and reliable.

The Null Object pattern can be used for reducing code complexity by managing optional dependencies with default behaviour as discussed in the Reject the Null Checked Object post.

This post aims to illustrate the problem with the Null Object pattern and how to resolve it using a simple lambda expression.

The problem is that the null check pattern can potentially lead to performance degradation and unnecessary failure.

The solution is to avoid executing unnecessary operations especially for default behaviour.

Setup

The intent of the sample application is to read and parse XML documents for a book store. The document reader is responsible for logging the book titles using the code below.

    public interface ILogger
    {
        void Log(string message);
    }

    public class ConsoleLogger : ILogger
    {
        public void Log(string message)
        {
            Console.WriteLine(message);
        }
    }

    public class DocumentReader
    {
        private readonly ILogger _logger;

        public DocumentReader(ILogger logger)
        {
            if (logger == null) throw new ArgumentNullException("logger");
            _logger = logger;
        }

        public void Read(XmlDocument document)
        {
            if (document == null) throw new ArgumentNullException("document");
            var books = document.SelectNodes("catalog/book");
            if (books == null) throw new XmlException("Catalog/book missing.");

            _logger.Log(string.Format("Titles: {0}.", 
                        string.Join(", ", GetBookTitles(document))));
            
            // Implementation
        }

        private static IEnumerable<string> GetBookTitles(XmlNode document)
        {
            Console.WriteLine("Retrieving the book titles");
            var titlesNodes = document.SelectNodes("catalog/book/title");
            if (titlesNodes == null) yield break;
            foreach (XmlElement title in titlesNodes)
            {
                yield return title.InnerText;
            }
        }
    }

Problem

Here is an example that illustrates the execution of the application.

 

        static void Main(string[] args)
        {
            var document = new XmlDocument();
            document.LoadXml(@"<catalog>
                                 <book><title>Developer's Guide</title></book>
                                 <book><title>Tester's Guide</title></book>
                               </catalog>");

            var logger = new ConsoleLogger();
            var docReader = new DocumentReader(logger);

            docReader.Read(document);
            Console.ReadLine();
        }

Retrieving the book titles
Book titles: Developer's Guide, Tester's Guide.

 
    public class NullLogger : ILogger
    {
        public void Log(string message)
        {
            // Purposefully provides no behaviour
        }
    }

    var logger = new NullLogger();
    var docReader = new DocumentReader(logger);

    docReader.Read(document);
    Console.ReadLine();

Retrieving the book titles

Solution

Here is an example that illustrates the improved version. The logger was modified to accept two methods. The first method takes a string for simple logging operations and the second method takes a lambda function that will produce a string.

 
    public interface ILogger
    {
        void Log(string message);
        void Log(Func<string> messageFunc);
    }

    public class NullLogger : ILogger
    {
        public void Log(string message)
        {
            // Purposefully provides no behaviour
        }

        public void Log(Func<string> messageFunc)
        {
            // Purposefully provides no behaviour
        }
    }

    public class ConsoleLogger : ILogger
    {
        public void Log(string message)
        {
            Console.WriteLine(message);
        }

        public void Log(Func<string> messageFunc)
        {
            try
            {
                Console.WriteLine(messageFunc.Invoke());
            }
            catch (Exception ex)
            {
                Console.WriteLine("Failed to log the result. Error: {0}", ex);
            } 
        }
    }

    public class DocumentReader
    {
        public void Read(XmlDocument document)
        {
            _logger.Log(() => string.Format("Titles: {0}.", 
                              string.Join(", ", GetBookTitles(document))));
           ...
         }
    }

Running the example using the Console Logger will produce the same result as the original example.

Let's run the Null Logger example again.

 
    var logger = new NullLogger();
    var docReader = new DocumentReader(logger);

    docReader.Read(document);
    Console.ReadLine();

Summary

The null object checking pattern simplifies solutions by removing the need to check for null. The drawback is the potential risk of impeding performance and causing failure. Passing around lamba expression functions can be a subtle solution to overcome the problem without overcomplicating the code.

Software solutions become overcomplicated with dozens of long and ugly checks for null. It is like a pothole and you will eventually fall in when you’re not looking carefully.

The problem is that methods returns null instead of real objects and every client must check for null to avoid the application from blowing up due to a NullReferenceExcetion (Object reference not set to an instance of an object).

The solution is to return a null object that exhibits default behaviour. This is called the Null Object design pattern.

Download Source Code

Example 1: Debug Mode

Here is an example where additional tasks are performed such as logging verbose messages when the application is in debug mode.

The cyclomatic complexity of the application has increased due to the number checks as shown below.

Null Checking

            if (isDebugMode)
            {
                logger.Log("Saving - Step 1");
            }

            // Save Step 1

            if (isDebugMode)
            {
                logger.Log("Saving - Step 2");
            }

            // Save Step 2

Null Object Design Pattern

            
public class NullLogger : ILogger
{
    public void Log(string message)
    {
        // Purposefully provides no behaviour
    }
}

logger.Log("Saving - Step 1");
// Save Step 1
logger.Log("Saving - Step 2");
// Save Step 2

Example 2: Registration System

The use case is to create a service to confirm reservations and to send optional confirmation notices. The service is also responsible for retrieving reservations that can be filtered based on the confirmation status.

The following interfaces will be used in the illustration.

    public interface INotification<in T>
    {
        void Notifiy(T message);
    }

    public interface IReservationRepository
    {
        void Save(Confirmation request);
        IEnumerable<Reservation> GetReservations();
    }

Null Checking

Here is the complicated example that performs null checks.

    public class ReservationServiceV1 : IReservationService
    {
        private readonly IReservationRepository _repository;
        private readonly INotification<Confirmation> _notifier;

        public ReservationServiceV1(IReservationRepository repository, 
                                    INotification<Confirmation> notifier)
        {
            if (repository == null) throw new ArgumentNullException("repository");
            _repository = repository;
            _notifier = notifier;
        }

        public void Confirm(Confirmation request)
        {
            _repository.Save(request);
            if (_notifier != null) _notifier.Notifiy(request);
        }

        public IEnumerable<Reservation> GetReservations(bool confirmed)
        {
            var reservations = _repository.GetReservations();
            return reservations == null ? null : 
                   reservations.Where(reservation => reservation.Confirmed == confirmed);
        }
    }

Null Object Design Pattern

Here is the simplified version that works with default behaviour.

 
public class NullNotification<T> : INotification<T>
{
    public void Notifiy(T message)
    {
        // Purposefully provides no behaviour
    }
}
   
public class ReservationServiceV2 : IReservationService
{
    private readonly IReservationRepository _repository;
    private readonly INotification<Confirmation> _notifier;

    public ReservationServiceV2(IReservationRepository repository, 
                                INotification<Confirmation> notifier)
    {
        if (repository == null) throw new ArgumentNullException("repository");
        if (notifier == null) throw new ArgumentNullException("notifier");
        _repository = repository;
        _notifier = notifier;
    }

    public void Confirm(Confirmation request)
    {
        _repository.Save(request);
        _notifier.Notifiy(request);
    }

    public IEnumerable<Reservation> GetReservations(bool confirmed)
    {
        return _repository.GetReservations()
                          .Where(reservation => reservation.Confirmed == confirmed);
    }
}

Summary

Avoid returning null and return default behaviour instead; such as an empty list. The Null Object design pattern will simplify code and reduce potential slip-ups causing unexpected failure.