Split fmtp on semicolons not spaces as per RFC6871

BUG=4617
R=pthatcher@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/47169004

Cr-Commit-Position: refs/heads/master@{#9193}
diff --git a/webrtc/base/stringencode_unittest.cc b/webrtc/base/stringencode_unittest.cc
index c9e726e..77fae35 100644
--- a/webrtc/base/stringencode_unittest.cc
+++ b/webrtc/base/stringencode_unittest.cc
@@ -298,6 +298,52 @@
   ASSERT_STREQ("E F", fields.at(3).c_str());
 }
 
+TEST(TokenizeFirstTest, NoLeadingSpaces) {
+  std::string token;
+  std::string rest;
+
+  ASSERT_TRUE(tokenize_first("A &*${}", ' ', &token, &rest));
+  ASSERT_STREQ("A", token.c_str());
+  ASSERT_STREQ("&*${}", rest.c_str());
+
+  ASSERT_TRUE(tokenize_first("A B& *${}", ' ', &token, &rest));
+  ASSERT_STREQ("A", token.c_str());
+  ASSERT_STREQ("B& *${}", rest.c_str());
+
+  ASSERT_TRUE(tokenize_first("A    B& *${}    ", ' ', &token, &rest));
+  ASSERT_STREQ("A", token.c_str());
+  ASSERT_STREQ("B& *${}    ", rest.c_str());
+}
+
+TEST(TokenizeFirstTest, LeadingSpaces) {
+  std::string token;
+  std::string rest;
+
+  ASSERT_TRUE(tokenize_first("     A B C", ' ', &token, &rest));
+  ASSERT_STREQ("", token.c_str());
+  ASSERT_STREQ("A B C", rest.c_str());
+
+  ASSERT_TRUE(tokenize_first("     A    B   C    ", ' ', &token, &rest));
+  ASSERT_STREQ("", token.c_str());
+  ASSERT_STREQ("A    B   C    ", rest.c_str());
+}
+
+TEST(TokenizeFirstTest, SingleToken) {
+  std::string token;
+  std::string rest;
+
+  // In the case where we cannot find delimiter the whole string is a token.
+  ASSERT_FALSE(tokenize_first("ABC", ' ', &token, &rest));
+
+  ASSERT_TRUE(tokenize_first("ABC    ", ' ', &token, &rest));
+  ASSERT_STREQ("ABC", token.c_str());
+  ASSERT_STREQ("", rest.c_str());
+
+  ASSERT_TRUE(tokenize_first("    ABC    ", ' ', &token, &rest));
+  ASSERT_STREQ("", token.c_str());
+  ASSERT_STREQ("ABC    ", rest.c_str());
+}
+
 // Tests counting substrings.
 TEST(SplitTest, CountSubstrings) {
   std::vector<std::string> fields;