C语言词法分析器

这是老师布置的编译原理的实验课任务,课余时间花了近一个星期的时间去编写代码(主要是C++太久没有用了,好多函数都不熟悉,查阅了很多资料),这次的词法分析只有简单的语法错误判断功能,约等于没有.如果想要增加功能可以在相关函数代码段增加即可.

而且token的种别码只做了粗略的区分,想要加上也很简单

c语言词法分析流程如图所示:


代码如下:这次的代码算是我对代码整洁之道前期知识的一个练习,特别是变量的命名以及函数的编写,如果觉得写的不错可以看看我关于代码整洁之道的相关文章
这次的代码基本没有写注释,但是我相信大家结合流程图也能看懂,如果觉得太长可以将代码复制到visual studio中将函数代码块闭合起来这样看起来就逻辑清楚了
#include <iostream>
#include <string>
#include <map>
#include <ctype.h>
#include <algorithm>
using namespace std;
string readFile(string fileName);
string fileFilter();
string singleLineCommentsFilter();
string multilineCommmentsFileter();
string specialCharacterFilter();
void separateAndJudge();
bool isReservedWord(string vocabulary);
void separateAndJudge();
void showTokenData();
int digitStarted(int cnt);
bool isBoundSymbol(char ch);
bool isOperator(char ch);
int judgeStartingCharactorType(char ch);
bool isDigit(char ch);
bool isAlpha(char ch);
int alphaStarted(int cnt);
int underlineStarted(int cnt);
string transCharToString(char ch);
int operatorStarted(int cnt);
string codeSource;
map<string, int> tokens;//identifier is 1,reservedWord 2,digit 3,borderSymbol 4,operator 5
int main()
{
	codeSource = readFile("d:\\testCode.txt");
	cout << "This is source code" << endl << "---------------------------------" << endl << endl;
	cout << codeSource << endl;

	cout << "This is code filtered" << endl << "---------------------------------" << endl;
	codeSource = fileFilter();
	cout << codeSource << endl;
	cout << "-----------------------" << endl;


	separateAndJudge();

	cout << "this is tokens " << endl;
	cout << "-----------------------" << endl;
	showTokenData();
	return 0;
}


string readFile(string fileName)
{
	FILE* fp;
	if ((fp = fopen(fileName.c_str(), "r")) == NULL)
	{
		cout << "cant open file";
		exit(0);
	}
	else
	{
		string codeSource;
		char ch;
		while ((ch = fgetc(fp)) != EOF)
		{
			codeSource += ch;
		}
		return codeSource;
	}
};

string fileFilter()
{
	string filteredCode = singleLineCommentsFilter();
	filteredCode = multilineCommmentsFileter();
	filteredCode = specialCharacterFilter();
	return filteredCode;
};
string singleLineCommentsFilter()
{
	long cnt = 0;
	for (; cnt < codeSource.length(); cnt++)
	{
		while (codeSource[cnt] == '/' and codeSource[cnt + 1] == '/' and cnt < codeSource.length())
		{
			while (codeSource[cnt] != '\n')
			{
				codeSource.erase(cnt, 1);
			}
		}
	}
	return codeSource;
}
string multilineCommmentsFileter()
{
	int cnt = 0;
	for (; cnt < codeSource.length(); cnt++)
	{
		if (codeSource[cnt] == '/' and codeSource[cnt + 1] == '*' )
		{
		
				do
				{
					
					codeSource.erase(cnt, 1);
					if ((codeSource[cnt + 1] == '*' and codeSource[cnt + 2] != '/'))
					{
						cout << "multilineCommments unmatch"<<endl;
						exit(0);
					};

				} while ((codeSource[cnt + 1] != '*' and codeSource[cnt + 2] != '/'));
				codeSource.erase(cnt, 4);
			
		}

	}
	return codeSource;
}
string specialCharacterFilter()
{
	for (int cnt = 0; cnt < codeSource.length(); cnt++)
	{
		if (codeSource[cnt] == '\n' or codeSource[cnt] == '\t' or codeSource[cnt] == '\v' or codeSource[cnt] == '\r')
		{
			codeSource.erase(cnt, 1);
			cnt--;
		}
	}
	return codeSource;
}

void separateAndJudge()
{
	int cnt = 0;
	for (; cnt < codeSource.length(); cnt++)
	{
		int nowCnt = 0;
		while (codeSource[cnt] != ' ' and cnt < codeSource.length())
		{
			string a = "";
			switch (judgeStartingCharactorType(codeSource[cnt]))
			{
			case 1:
				cnt = digitStarted(cnt);
				break;
			case 2:
				cnt = alphaStarted(cnt);
				break;
			case 3:
				cnt = underlineStarted(cnt);
				break;
			case 4:
				tokens[transCharToString(codeSource[cnt])] = 4;
				cnt++;
				break;
			case 5:
				cnt = operatorStarted(cnt);
				//tokens[transCharToString(codeSource[cnt])] = 5;
				//cnt++;
				break;
			case 6:
				cout << "unrecognizable charactor!!!" << endl;
				cout << "please check grammer again." << endl;
				exit(0);
				cnt++;
				break;
			default:
				cnt++;
				break;
			}

		}
	}
}
int judgeStartingCharactorType(char ch)
{
	int type = 0;
	if (isDigit(ch)) { type = 1; }
	else
	{
		if (isAlpha(ch)) { type = 2; }
		else
		{
			if (ch == '_') { type = 3; }
			else
			{
				if (isBoundSymbol(ch)) { type = 4; }
				else
				{
					if (isOperator(ch)) { type = 5; }
					else { type = 6; }
				}
			}
		}
	}
	return type;
}
bool isBoundSymbol(char ch)
{
	string temp = "";
	temp += ch;
	bool flag = false;
	string boundSymbol[7] =
	{
		"(",   ")",   ",",   ";",  "{",  "}","\"",
	};
	for (int i = 0; i < 7; i++)
	{
		if (boundSymbol[i] == temp)
		{
			flag = true;
		}
	}
	return flag;
}
bool isOperator(char ch)
{
	string temp = transCharToString(ch);
	bool flag = false;
	string operators[9]=
	{
		"+","-","*","/","=","%",">","<","=", 
	};
	for (int i = 0; i < 9; i++)
	{
		if (operators[i] == temp)
		{
			flag = true;
		}
	}
	return flag;
}
bool isDigit(char ch)
{
	bool flag = false;
	if (ch >= '0' and ch <= '9')
	{
		flag = true;
	}
	return flag;
}
bool isAlpha(char ch)
{
	bool flag = false;
	if ((ch >= 'a' and ch <= 'z') or (ch >= 'A' and ch <= 'Z'))
	{
		flag = true;
	};
	return flag;
}
int digitStarted(int cnt)
{
	string digit;
	digit += codeSource[cnt];
	cnt++;
	while (isDigit(codeSource[cnt]) or codeSource[cnt] == '.')
	{
		digit += codeSource[cnt];
		++cnt;
	}
	tokens[digit] = 3;
	return cnt;
}
int alphaStarted(int cnt)
{
	string alpha;
	alpha += codeSource[cnt];
	cnt++;
	while (isAlpha(codeSource[cnt]) or isDigit(codeSource[cnt]) or codeSource[cnt] == '_')
	{
		alpha += codeSource[cnt];
		++cnt;
	}
	if (isReservedWord(alpha)) { tokens[alpha] = 2; }
	else { tokens[alpha] = 1; }
	return cnt;
}
int underlineStarted(int cnt)
{
	string word;
	word += codeSource[cnt];
	cnt++;
	while (isAlpha(codeSource[cnt]) or isDigit(codeSource[cnt]))
	{
		word += codeSource[cnt];
		++cnt;
	}
	tokens[word] = 1;
	return cnt;
}
int operatorStarted(int cnt)
{
	bool flag = false;
	string operators = "";
	
	if (codeSource[cnt]=='=' and codeSource[cnt+1]=='=' )
	{
		operators = "==";
		cnt++;	
	}
	else
	{
		operators = transCharToString(codeSource[cnt]);
		cnt++;
	}
	tokens[operators] = 5;
	return cnt;
}

string transCharToString(char ch)
{
	string temp = " ";
	temp[0] = ch;
	return temp;
}
bool isReservedWord(string vocabulary)
{
	string reserveWords[32] = {
	 "auto", "break", "case", "char", "const", "continue",
	"default", "do", "double", "else", "enum", "extern",
	"float", "for", "goto", "if", "int", "long",
	"register", "return", "short", "signed", "sizeof", "static",
	"struct", "switch", "typedef", "union", "unsigned", "void",
	 "volatile", "while"
	};
	bool flag = false;
	for (int i = 0; i < 32; i++)
	{
		if (reserveWords[i] == vocabulary)
		{
			flag = true;
		}
	}
	return flag;
};


void showTokenData()
{

	map<string, int>::iterator iter;

	for (iter = tokens.begin(); iter != tokens.end(); iter++)

		cout <<"<"<< iter->first << ',' << iter->second<<">" << endl;
}

程序设计语言与编译实验：C语言词法分析器的设计与实现

一、实验目的及要求

本次实验通过用C语言设计、编制、调试一个词法分析子程序，识别单词，实现一个C语言词法分析器，经过此过程可以加深对编译器解析单词流的过程的了解。

运行环境：

硬件：windows 10

软件：Code::Blocks 17.12
二、实验步骤

1.查询资料，了解词法分析器的工作过程与原理。

2.分析题目，整理出基本设计思路。

3.实践编码，将设计思想转换成C语言编码实现，编译运行。

4.测试功能，自己选取C 语言的一个适当大小的子集，多次测试，查看运行结果，检测改分析器的分析结果是否正确。
三、实验内容

1.源代码
#include<string.h>
#include<stdio.h>
#include<stdlib.h>
#include<ctype.h>
//定义关键字
char *Key[10]={"main","void","int","char","printf","scanf","else","if","return"};
char Word[20],ch;      //存储识别出的单词流
int IsAlpha(char c)      //判断是否为字母
{              
     if(((c<='z')&&(c>='a'))||((c<='Z')&&(c>='A')))
         return 1;
    else 
        return 0;
}

int IsNum(char c)           //判断是否为数字
{               
     if(c>='0'&&c<='9') 
         return 1;
    else 
         return 0;
 }

int IsKey(char *Word)                 //识别关键字函数
{         
     int m,i;
     for(i=0;i<9;i++)
     {
        if((m=strcmp(Word,Key[i]))==0)
      {
        if(i==0)
            return 2;
        else
            return 1;
       }
    }
     return 0;
}

void scanner(FILE *fp)     //扫描函数
{        
char Word[20]={'\0'};
char ch;
int i,c;
ch=fgetc(fp);       //获取字符，指针fp并自动指向下一个字符
if(IsAlpha(ch))            //判断该字符是否是字母{            
Word[0]=ch;
ch=fgetc(fp);
i=1;
while(IsNum(ch)||IsAlpha(ch))   //判断该字符是否是字母或数字{  
Word[i]=ch;
i++;
ch=fgetc(fp);
}
Word[i]='\0';      //'\0' 代表字符结束(空格)
fseek(fp,-1,1);                       
c=IsKey(Word);     //判断是否是关键字                     
if(c==0) printf("%s\t$普通标识符\n\n",Word);//不是关键字
else if(c==2)     printf("%s\t$主函数\n\n",Word);
else    printf("%s\t$关键字\n\n",Word);      //输出关键字
}

else              //开始判断的字符不是字母
{
   if(IsNum(ch))                //判断是否是数字
   {                
        Word[0]=ch;
        ch=fgetc(fp);
        i=1;
        while(IsNum(ch))
        {
           Word[i]=ch;
            i++;
            ch=fgetc(fp);
        }
        Word[i]='\0';
        fseek(fp,-1,1);    //回退
        printf("%s\t$无符号实数\n\n",Word);}

        else        //开始判断的字符不是字母也不是数字
      {
         Word[0]=ch;
         switch(ch)
         {
           case'[':

          case']':

          case'(':

          case')':

          case'{':

          case'}':

          case',':

          case'"':

          case';':printf("%s\t$界符\n\n",Word); break;

         case'+':ch=fgetc(fp);

        Word[1]=ch;

         if(ch=='=')
         {
             printf("%s\t$运算符\n\n",Word);//运算符“+=”
          }

         else if(ch=='+')
        {
            printf("%s\t$运算符\n\n",Word); //判断结果为“++”
        }

         else
        {
           fseek(fp,-1,1);
           printf(“%s\t$运算符\n\n”,Word); //判断结果为“+”
         }
         break;

         case'-':ch=fgetc(fp);
         Word[1]=ch;
         if(ch=='=')
         {
             printf(“%s\t$运算\n\n”,Word);                     
         }
        else if(ch=='-')
        {
           printf("%s\t$运算符\n\n",Word); //判断结果为“--”
        }
       else
        {
           fseek(fp,-1,1);
           printf("%s\t$运算符\n\n",Word); //判断结果为“-”
        }
        break;
        case'*':
        case'/':
        case'!':
        case'=':ch=fgetc(fp);
        if(ch=='=')
        {
           printf("%s\t$运算符\n\n",Word);
        }
        else 
        {
             fseek(fp,-1,1);
             printf("%s\t$运算符\n\n",Word);
       }
       break;

      case'<':ch=fgetc(fp);
      Word[1]=ch;
      if(ch=='=')
      {
         printf("%s\t$运算符\n\n",Word); //判断结果为运算符“<=”
      }

        else if(ch=='<')
        {
           printf("%s\t$运算符\n\n",Word); //判断结果为“<<”
        }

        else
         {
            fseek(fp,-1,1);
            printf("%s\t$运算符\n\n",Word); //判断结果为“<”
          }
         break;

       case'>':ch=fgetc(fp);
       Word[1]=ch;
       if(ch=='=') printf("%s\t$运算符\n\n",Word);
       else 
       {
          fseek(fp,-1,1);
          printf("%s\t$运算符\n\n",Word);
        }
        break;

       case'%':ch=fgetc(fp);
       Word[1]=ch;
       if(ch=='='){printf("%s\t$运算符\n\n",Word);}
       if(IsAlpha(ch)) printf("%s\t$类型标识符\n\n",Word);
       else
      {
         fseek(fp,-1,1);
         printf("%s\t$取余运算符\n\n",Word);
       }
          break;
          default:printf("无法识别字符!\n\n"); break;}
    }
}

main()
{
char in_fn[30];  //文件路径              
FILE *fp;
printf("\n请输入源文件名（包括路径和后缀名）:");
while(1){
gets(in_fn);
scanf("%s",in_fn);
if((fp=fopen(in_fn,"r"))!=NULL)  break;  //读取文件内容，并返回文件指针，该指针指向文件的第一个字符

else printf("文件路径错误!请重新输入:");}
    printf("\n******************* 词法分析结果下 *******************\n");
do{
ch=fgetc(fp);
if(ch=='#')  break;      //文件以#结尾，作为扫描结束条件

else if(ch==' '||ch=='\t'||ch=='\n'){} //忽略空格，空白，和换行
else{
fseek(fp,-1,1);          //回退一个字节开始识别单词流
scanner(fp);
}}while(ch!='#');return0;} 

四、实验结果

1.解析源文件

int main(）

{

float b;

float a=5;

b=a++;

printf("%c",b);

return;

}#
2.实验结果

Java编写的C语言词法分析器



                


    这是java编写的C语言词法分析器，我也是参考很多代码，然后将核心代码整理起来，准备放在QQ空间和博客上，目的是互相学习借鉴，希望可以得到高手改进。这个词法分析器实现的功能有打开文件、保存文件、打开帮助文档、文本域内容的剪切和复制和黏贴、进行词法分析


程序的项目结构如图，Word类和Unidentifiable类是两个JavaBean类，存放的参数有两个row(整型)、word(String)，row用于获取行数，word用于获取标识符，LexerFrame是词法分析器的界面类，Analyze封装了进行词法分析的核心代码 ，doc文件夹放一个帮助文档，当用户点击帮助按钮时可以弹出来以帮助用户使用。


Github项目链接：https://github.com/u014427391/lexer1.1.0，欢迎star



//核心程序：



package com.lexer;

import java.util.ArrayList;


/**
*1~20号为关键字,用下标表示,i+1就是其机器码;21~40号为操作符,用下标表示,i+21就是其机器码;41~60号为分界符,
*  用下标表示,i+41就是其机器码;用户自定义的标识符，其机器码为51;常数的机器码为52；不可以识别的标识符，其机器码为0
*/ 

public class Analyze {

 //关键字
 private String keyword[]={"int","long","char","if","else","for","while","return","break","continue",
   "switch","case","default","float","double","void","struct","static","do","short"};
 
 //运算符
 private String operator[]={"+","-","*","/","%","=",">","<","!","==","!=",">=","<=","++","--","&","&&","||","[","]"};
 
 //分界符
 private String delimiter[]={",",";","(",")","{","}","\'","\"",":","#"};
 
 public Analyze() {
  
 }
 
 /**
  * 判断是否是数字
  */
 public boolean isDigit(char ch){
  if(ch>='0'&&ch<='9'){
   return true;
  }else{
   return false;
  }
 }
 
 /**
  * 判断是否是字母的函数
  */
 public boolean isLetter(char ch){
  if((ch>='a'&&ch<='z')||(ch>='A'&&ch<='Z')){
   return true;
  }else{
   return false;
  }
 }
 
 /**
  * 判断是否由两个运算符组成
  */
 public boolean isTwoOperator(String str,char ch){
  char lc;
  int flag=0;
  if(str.length()>1||str.length()==0){//字符数大于2和无字符的情况
   return false;
  }else{//字符数等于2的情况
   lc=str.charAt(str.length()-1);
   if(ch=='='&&(lc=='>'||lc=='<'||lc=='='||lc=='!')){
    
   }else if(ch=='+'&&lc=='+'){
    
   }else if(ch=='-'&&lc=='-'){
    
   }else if(ch=='|'&&lc=='|'){
    
   }else if(ch=='&'&&lc=='&'){
    
   }else{
    return false;//否就返回false
   }
   return true;//其它符号的情况都返回true
  }
 }
 
 /**
  * 获取关键字的机器码
  */
 public int getKeywordOpcodes(String str){
  
  int i; 
  for(i=0;i<keyword.length;i++){
   if(str.equals(keyword[i]))
    break;
  }
  if(i<keyword.length){
   return i+1;//返回关键字的机器码
  }else{
   return 0;
  }
 }
 
 /**
  * 获取操作符的机器码
  */
 public int getOperatorOpcodes(String str){
  int i;
  for(i=0;i<operator.length;i++){
   if(str.equals(operator[i]))
    break;
  }
  
  if(i<operator.length)
   return i+21;//返回操作符的机器码
  else
   return 0;
 }
 
 /**
  * 获取分界符的机器码
  */
 public int getDelimiterOpcodes(String str){
  int i;
  for(i=0;i<delimiter.length;i++){
   if(str.equals(delimiter[i]))
    break;
  }
  
  if(i<delimiter.length)
   return i+41;//返回分界符的机器码
  else
   return 0;
 }
  
 /**
  * 判断字符是否可以识别
  */
 public boolean isIdent(String str){
  char ch;
  int i;
  for(i=0;i<str.length();i++){
   ch=str.charAt(i);
   //非数字串的情况和非由英文字母组成的字符串
   if((i==0&&!isLetter(ch))||(!isDigit(ch)&&!isLetter(ch))){
    break;
   }
  }
  
  if(i<str.length()){
   return false;
     }else{
   return true;
     }
 }
 
 /**
  *
  * 预处理函数
  */
 public String preFunction(String str){
  String ts="";
  
  int i;
  
  char ch,nc;
  //这里的i<str.length()-1
  for(i=0;i<str.length()-1;i++){ 
   ch=str.charAt(i);
   nc=str.charAt(i+1);
   
   if(ch=='\n'){//如果字符是换行符,将\n换成$
    ch='$';
    ts=ts+ch;
   }else if(ch==' '||ch=='\r'||ch=='\t'){
    if(nc==' '||nc=='\r'||ch=='\t'){
     continue;//连续' '或者'\t'或者'\r'的情况，直接跳过
    }else{
     ch=' ';//一个' '或者'\t'或者'\r'的情况，将这些字符换成' '
     ts=ts+ch;
    }
   }else{
    ts=ts+ch;//将字符连起来
   }
  }
  ch=str.charAt(str.length()-1);
  if(ch!=' '&&ch!='\r'&&ch!='\t'&&ch!='\n'){
   ts=ts+ch;
  }
  return ts;
 }
 
 /**
  * 将字符串分成一个个单词，存放在数组列表
  */
 public ArrayList<Word> divide(String str){
  ArrayList<Word> list=new ArrayList<Word>();
  
  String s="";
  char ch;
  int i;
  int row=1;
  
  for(i=0;i<str.length();i++){
   ch=str.charAt(i);
   if(i==0&&ch==' ')//字符串的第一个字符
    continue;
   if(ch==' '){//' '或者'\t'或者'\r'的情况
    if(s!=""){
     list.add(new Word(row, s));
     s="";//置空
    }else{
     continue;
    }
   }else if(isDigit(ch)||isLetter(ch)){
    if(s==""||isDigit(s.charAt(s.length()-1))||isLetter(s.charAt(s.length()-1))){
     s = s + ch;
    }else{
     list.add(new Word(row, s));
     s = "";
     s=s + ch;
    } 
   }else{
    if(isTwoOperator(s, ch)){//两个运算符的情况
     s = s + ch;
    }else{
     if(s==""&&ch!='$'){
      s = s + ch;
     }else if(s==""&&ch=='$'){//若检测到$符号，就换行
      row++;//行数加一
     }else{
      list.add(new Word(row, s));
      s = "";
      if(ch!='$'){
       s=s + ch;
      }else{
       row++;
      }
     }
    } 
   }
  }
  if(s!=""){
   list.add(new Word(row, s));
  }
  return list;
 }
 
 /**
  * 判断字符串是数字串，单个字符，还是一个字符串
  */
 public int check(String str){
  char ch;
  ch=str.charAt(0);
  if(ch>='0'&&ch<='9'){
   return 1;//数字串
  }
  if(str.length()==1)
   return 2;//单个字符
  else
   return 3;//一个字符串
 }
 
 /**
  *
  * 检查字符串是否为数字串，返回其机器码
  */
 public int checkDigit(String str){
  int i;
  char ch;
  for(i=0;i<str.length();i++){
   ch=str.charAt(i);
   if(ch>'9'||ch<'0')
    break;
  }
  if(i<str.length()){
   return 0;//不可识别的情况
  }else{
   return 52;//常数
  }
 }
 
 /**
  *
  * 检查字符串是否为单个字符，返回其机器码
  */
 public int checkChar(String str){
  if(getOperatorOpcodes(str)!=0){//操作符
   return getOperatorOpcodes(str);
  }else if(getDelimiterOpcodes(str)!=0){//分界符
   return getDelimiterOpcodes(str);
  }else if(isIdent(str)){
   return 51;//用户自定义标识符的机器码
  }else{
   return 0;//不可以被识别的标识符，机器码为0
  }
 }
 
 /**
  *
  * 检查字符串是否为字符串，返回其机器码
  */
 public int checkString(String str){
  if(getOperatorOpcodes(str)!=0){//操作符
   return getOperatorOpcodes(str);
  }else if(getKeywordOpcodes(str)!=0){//关键字
   return getKeywordOpcodes(str);
    }else if(isIdent(str)){
   return 51;//用户自定义标识符的机器码
    }else{
   return 0;//不可以被识别的标识符，机器码为0
    }
 }
 
 
}




//********************************************************************************************************************
在界面类写这个方法来调用方法

/**
  * 词法分析
  */
 public void doTokenizing(){
  
  consoleTextArea.setText(null);
  ArrayList<Word> wlist=new ArrayList<Word>();
  ArrayList<Unidentifiable> ulist=new ArrayList<Unidentifiable>();
  
  String s,ts,str;
  Word word;
  
  int i;
  int opcodes=-1;
  int errorNum=0;
  
  int count=0;
  
  s=fileContentTextArea.getText();
  if(s.length()>1){
   ts=analyze.preFunction(s);
   wlist=analyze.divide(ts);
   values=new String[wlist.size()][3];
   while(wlist.size()>0){
    word=(Word)wlist.remove(0);
    str=word.getWord();
    i=analyze.check(str);
    switch (i) {
    case 1:
     opcodes=analyze.checkDigit(str);
     break;
    case 2:
     opcodes=analyze.checkChar(str);
     break;
    case 3:
     opcodes=analyze.checkString(str);
     break;
    }
    
    if(opcodes==0){
     Unidentifiable u=new Unidentifiable(word.getRow(), str);
     ulist.add(u);
     errorNum++;
    }
    
    values[count][0]=String.valueOf(word.getRow());
    values[count][1]=str;
    values[count][2]=String.valueOf(opcodes);
    
    count++;
    
   }
   
   //更新表格内容
   DefaultTableModel model=(DefaultTableModel)table.getModel();
   while(model.getRowCount()>0){
    model.removeRow(model.getRowCount()-1);
   }
      
        model.setDataVector(values,title);
   
   table=new JTable(model);
    
    consoleTextArea.append("共有"+errorNum+"处错误!"+"\n");
    while (ulist.size()>0) {
     int r;
     String string;
     Unidentifiable uni=ulist.remove(0);
     r=uni.getRow();
     string=uni.getWord();
     consoleTextArea.append("第"+r+"行:"+"错误,"+string+"\n");
    }
  }else{
   int j;
   j=JOptionPane.showConfirmDialog(this, "请输入程序!");
   if(j!=JOptionPane.YES_OPTION){
    return;
   }
  }
  
 }



 

C语言词法分析器(C++语言实现)

这是老师布置的编译原理的实验课任务,课余时间花了近一个星期的时间去编写代码(主要是C++太久没有用了,好多函数都不熟悉,查阅了很多资料),这次的词法分析也没有语法错误判断功能,如果想要增加功能可以在相关函数代码段增加即可.对于此法单元的种别码也只是简单区分,不过自己加上去就可以了

c语言词法分析流程如图所示:

代码如下:这次的代码算是我对代码整洁之道前期知识的一个练习,特别是变量的命名以及函数的编写,如果觉得写的不错可以看看我关于代码整洁之道的相关文章
这次的代码基本没有写注释,但是我相信大家结合流程图也能看懂,如果觉得太长可以将代码复制到visual studio中将函数代码块闭合起来这样看起来就逻辑清楚了
#include <iostream>
#include <string>
#include <map>
#include <ctype.h>
#include <algorithm>
using namespace std;
string readFile(string fileName);
string fileFilter();
string singleLineCommentsFilter();
string multilineCommmentsFileter();
string specialCharacterFilter();
void separateAndJudge();

bool isReservedWord(string vocabulary);
void separateAndJudge();
void showTokenData();
int digitStarted(int cnt);
bool isBoundSymbol(char ch);
bool isOperator(char ch);
int judgeStartingCharactorType(char ch);
bool isDigit(char ch);
bool isAlpha(char ch);
int alphaStarted(int cnt);
int underlineStarted(int cnt);
string transCharToString(char ch);
string codeSource;
map<string, int> tokens;//identifier is 1,reservedWord 2,digit 3,borderSymbol 4,operator 5
int main()
{
	codeSource = readFile("h:\\testCode.txt");
	cout << "This is source code" << endl << "---------------------------------" << endl << endl;
	cout << codeSource << endl;

	cout << "This is code filtered" << endl << "---------------------------------" << endl ;
	codeSource = fileFilter();
	codeSource = fileFilter();
	cout << codeSource << endl;

	separateAndJudge();

	cout << "this is tokens " << endl;
	showTokenData();
	return 0;
}


string readFile(string fileName)
{
	FILE* fp;
	if ((fp = fopen(fileName.c_str(), "r")) == NULL)
	{
		cout << "cant open file";
		exit(0);
	}
	else
	{
		string codeSource;
		char ch;
		while ((ch = fgetc(fp)) != EOF)
		{
			codeSource += ch;
		}
		return codeSource;
	}
};
string fileFilter()
{
	string filteredCode = singleLineCommentsFilter();
	filteredCode = multilineCommmentsFileter();
	filteredCode = specialCharacterFilter();
	return filteredCode;
};
void separateAndJudge()
{
	int cnt = 0;
	for (; cnt < codeSource.length(); cnt++)
	{
		int nowCnt = 0;
		while (codeSource[cnt] != ' ' and cnt < codeSource.length())
		{
			string a = "";
			switch (judgeStartingCharactorType(codeSource[cnt]))
			{
			case 1:
				cnt = digitStarted(cnt);
				break;
			case 2:
				cnt = alphaStarted(cnt);
				break;
			case 3:
				cnt = underlineStarted(cnt);
				break;
			case 4:		
				tokens[transCharToString(codeSource[cnt])] = 4;
				cnt++;
				break;
			case 5:				
				tokens[transCharToString(codeSource[cnt])] = 5;
				cnt++;
				break;
			case 6:
				cout << "wrong grammer" << endl;
				exit(0);
				cnt++;
				break;
			default:
				cnt++;
				break;
			}
			
		}
	}
}
void showTokenData()
{
	
	map<string, int>::iterator iter;

	for (iter = tokens.begin(); iter != tokens.end(); iter++)

		cout << iter->first << ' ' << iter->second << endl;
}


string singleLineCommentsFilter()
{
	long cnt = 0;
	for (; cnt < codeSource.length(); cnt++)
	{
		while (codeSource[cnt] == '/' and codeSource[cnt + 1] == '/' and cnt < codeSource.length())
		{
			while (codeSource[cnt] != '\n')
			{
				codeSource.erase(cnt, 1);
			}
		}
	}
	return codeSource;
}
string multilineCommmentsFileter()
{
	int cnt = 0;
	for (; cnt < codeSource.length(); cnt++)
	{
		if (codeSource[cnt] == '/' and codeSource[cnt + 1] == '*' and cnt < codeSource.length())
		{
			do
			{
				codeSource.erase(cnt, 1);
				if (codeSource[cnt+2]==EOF)
				{
					cout << "multilineCommments wrong" << endl;
					exit(0);
				}
			} while (codeSource[cnt + 2] != '*' and codeSource[cnt + 3] != '/');
			codeSource.erase(cnt, 4);
		}

	}
	return codeSource;
}
string specialCharacterFilter()
{
	for (int cnt = 0; cnt < codeSource.length(); cnt++)
	{
		if (codeSource[cnt] == '\n' or codeSource[cnt] == '\t' or codeSource[cnt] == '\v' or codeSource[cnt] == '\r')
		{
			codeSource.erase(cnt, 1);
			cnt--;
		}
	}
	return codeSource;
}


int judgeStartingCharactorType(char ch)
{
	int type = 0;
	if (isDigit(ch)) { type = 1; }
	else
	{
		if (isAlpha(ch)) { type = 2; }
		else
		{
			if (ch == '_') { type = 3; }
			else
			{
				if (isBoundSymbol(ch)) { type = 4; }
				else
				{
					if (isOperator(ch)) { type = 5; }
					else { type = 6; }
				}
			}
		}
	}
	return type;
}

int digitStarted(int cnt)
{
	string digit;
	digit += codeSource[cnt];
	cnt++;
	while (isDigit(codeSource[cnt]) or codeSource[cnt] == '.')
	{
		digit += codeSource[cnt];
		++cnt;
	}
	tokens[digit] = 3;
	return cnt;
}
int alphaStarted(int cnt)
{
	string alpha;
	alpha += codeSource[cnt];
	cnt++;
	while (isAlpha(codeSource[cnt]) or isDigit(codeSource[cnt])  or codeSource[cnt]=='_')
	{
		alpha += codeSource[cnt];
		++cnt;
	}
	if (isReservedWord(alpha)) { tokens[alpha] = 2; }
	else { tokens[alpha] = 1; }
	return cnt;
}
int underlineStarted(int cnt)
{
	string word;
	word += codeSource[cnt];
	cnt++;
	while (isAlpha(codeSource[cnt]) or isDigit(codeSource[cnt]))
	{
		word += codeSource[cnt];
		++cnt;
	}
	tokens[word] = 1;
	return cnt;
}


string transCharToString(char ch)
{
	string temp = " ";
	temp[0] = ch;
	return temp;
}


bool isReservedWord(string vocabulary)
{
	string reserveWords[32] = {
	 "auto", "break", "case", "char", "const", "continue",
	"default", "do", "double", "else", "enum", "extern",
	"float", "for", "goto", "if", "int", "long",
	"register", "return", "short", "signed", "sizeof", "static",
	"struct", "switch", "typedef", "union", "unsigned", "void",
	 "volatile", "while"
	};
	bool flag = false;
	for (int i = 0; i <32 ; i++)
	{
		if (reserveWords[i]==vocabulary)
		{
			flag = true;
		}
	}
	return flag;
};

bool isBoundSymbol(char ch)
{
	string temp = "";
	temp += ch;
	bool flag = false;
	string boundSymbol[6] =
	{
		"(",   ")",   ",",   ";",  "{",  "}"
	};
	for (int i = 0; i < 6; i++)
	{
		if (boundSymbol[i] == temp)
		{
			flag = true;
		}
	}
	return flag;
}
bool isOperator(char ch)
{
	string temp = to_string(ch);
	bool flag = false;
	string operators[9] =
	{
		"+","-","*","/","=","%",">","<","=",
	};
	for (int i = 0; i < 9; i++)
	{
		if (operators[i] == temp)
		{
			flag = true;
		}
	}
	return flag;
}
bool isDigit(char ch)
{
	bool flag = false;
	if (ch >= '0' and ch <= '9')
	{
		flag = true;
	}
	return flag;
}
bool isAlpha(char ch)
{
	bool flag = false;
	if ((ch >= 'a' and ch <= 'z') or (ch >= 'A' and ch <= 'Z'))
	{
		flag = true;
	};
	return flag;
}