Addeds docs to tokenize

tokenize.c

@@ -4,14 +4,25 @@
 
 #include "tokenize.h"
 
+/// Takes in the user input and tokenizes it
+///
+/// \param string user input
+/// \param output
+/// \return number of tokens
 int tokenize(char *string, char **output) {
+    // Make strtok thread safe
     char *strtok_save;
+    // Our Delimiter is a space
     char *del = " ";
+    // First call is special
     output[0] = strtok_r(string, del, &strtok_save);
     char *token = output[0];
+    // Number of tokens is one
     int i = 1;
     while (token != NULL) {
+        // String is NULL to keep getting tokens from the string pass in the first call of strtok_r
         token = strtok_r(NULL, del, &strtok_save);
+        // Set the output
         output[i] = token;
         i++;
     }