Category Archives: sql

How to set up SQL Server System Alerts

March 6, 2017Database Administrator, dba, sql, SQL Server, T-SQLalerts, How To, lesson, set up, setup, SQL Server, System, system alerts, tutorialTidbytez 1

In this article I will be sharing the TSQL Script I use to set up SQL Server System Alerts. Prerequisites for this script to work are you have database mail set up and working and have defined an operator to use.

This script sets up alerts of severity 16 and up. These are errors you want to alert on because they either represent a resource issue, an integrity issue, or a hardware issue. Severity 15 and down tends to refer to the database and deals with issues that are tied to syntax errors, violations of foreign keys, etc. These issues do not typically require an alert and can be viewed in the error log.

Alert meaning:

016 Miscellaneous User Error

017 Insufficient Resources

018 Nonfatal Internal Error Detected

019 SQL Server Error in Resource

020 SQL Server Fatal Error in Current Process

021 SQL Server Fatal Error in Database (dbid) Process

022 SQL Server Fatal Error Table Integrity Suspect

023 SQL Server Fatal Error: Database Integrity Suspect

024 Hardware Error

025 (no description)

823 I/O Failure

824 I/O Failure

825 I/O Subsystem Failing (Not completely failed)

Just do a globale find and replace of PLACE_HOLDER with the operator you have defined for the alert and run the script.

USE msdb;
GO

/*
CHANGE OPERATOR NAME: PLACE_HOLDER
(TIP: LOOK AT OPERATORS IN SSMS FOR LIST OF AVAILABLE OPERATORS)
*/
/*
[1] DELETE ALERTS IF THEY ALREADY EXIST
*/
IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 016'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 016';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 017'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 017';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 018'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 018';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 019'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 019';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 020'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 020';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 021'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 021';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 022'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 022';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 023'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 023';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 024'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 024';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Severity 025'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Severity 025';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Error Number 823'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Error Number 823';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Error Number 824'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Error Number 824';
END
GO

IF EXISTS (
		SELECT *
		FROM dbo.sysalerts
		WHERE NAME = 'Error Number 825'
		)
BEGIN
	EXEC dbo.sp_delete_alert @name = N'Error Number 825';
END
GO

/*
[2] CREATE ALERTS AND SET UP NOTIFICATIONS
*/
EXEC msdb.dbo.sp_add_alert @name = N'Severity 016'
	,@message_id = 0
	,@severity = 16
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 016'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 017'
	,@message_id = 0
	,@severity = 17
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 017'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 018'
	,@message_id = 0
	,@severity = 18
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 018'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 019'
	,@message_id = 0
	,@severity = 19
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 019'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 020'
	,@message_id = 0
	,@severity = 20
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 020'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 021'
	,@message_id = 0
	,@severity = 21
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 021'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 022'
	,@message_id = 0
	,@severity = 22
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 022'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 023'
	,@message_id = 0
	,@severity = 23
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 023'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 024'
	,@message_id = 0
	,@severity = 24
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 024'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Severity 025'
	,@message_id = 0
	,@severity = 25
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Severity 025'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Error Number 823'
	,@message_id = 823
	,@severity = 0
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Error Number 823'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Error Number 824'
	,@message_id = 824
	,@severity = 0
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Error Number 824'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

EXEC msdb.dbo.sp_add_alert @name = N'Error Number 825'
	,@message_id = 825
	,@severity = 0
	,@enabled = 1
	,@delay_between_responses = 60
	,@include_event_description_in = 1
	,@job_id = N'00000000-0000-0000-0000-000000000000';
GO

EXEC msdb.dbo.sp_add_notification @alert_name = N'Error Number 825'
	,@operator_name = N'PLACE_HOLDER'
	,@notification_method = 7;
GO

How to identify databases with Guest user enabled

November 8, 2016Database Administrator, dba, sql, SQL Server, T-SQLenabled, guest, guide, How To, How to identify databases with Guest user enabled, howto, identify, lesson, Microsoft, RAP, Risk Assessment Program, SQL Server, tutorial, userTidbytez 1

As best practice it is recommended to disable guest user in every user database, i.e. not master, msdb and tempdb, to improve the security of SQL Server. Guest user permits access to a database for any logins that are not mapped to a specific database user. The guest user cannot be dropped but it can be disabled by revoking the CONNECT permission.

Use the script below to identify which databases have guest user enabled.

USE master;
GO

DECLARE @database_name SYSNAME
	,@sqlcmd NVARCHAR(4000)

DECLARE databases_cursor CURSOR
FOR
SELECT NAME
FROM sys.databases
WHERE STATE IN (0)
	AND database_id > 4
ORDER BY NAME

CREATE TABLE #guest_users_enabled (
	database_name SYSNAME
	,user_name SYSNAME
	,permission_name NVARCHAR(128)
	,state_desc NVARCHAR(6)
	)

OPEN databases_cursor;

FETCH NEXT
FROM databases_cursor
INTO @database_name;

WHILE @@FETCH_STATUS = 0
BEGIN
	SET @sqlcmd = N'use ' + @database_name + ';

        insert into #guest_users_enabled

        SELECT ''' + @database_name + ''' as database_name, name,

        permission_name, state_desc

        FROM sys.database_principals dpr

        INNER JOIN sys.database_permissions dpe

        ON dpr.principal_id = dpe.grantee_principal_id

        WHERE name = ''guest'' AND permission_name = ''CONNECT'''

	EXEC sp_executesql @sqlcmd

	FETCH NEXT
	FROM databases_cursor
	INTO @database_name;
END

SELECT database_name
	,user_name
	,permission_name
	,state_desc
FROM #guest_users_enabled
ORDER BY database_name ASC

DROP TABLE #guest_users_enabled

CLOSE databases_cursor;

DEALLOCATE databases_cursor;
GO

Use the script below, changing the placeholder database with the database identified in the script above, to revoke permission for the guest user to connect to that database.

USE [database name];

GO

REVOKE CONNECT FROM GUEST;

GO

How to resize TempDB in SQL Server

November 7, 2016Database Administrator, dba, sql, SQL Server, T-SQLguide, How To, initial size, lesson, Microsoft, RAP, resize, Risk Assessment Program, tempdb, TempDB initial database Size may be too small, tutorialTidbytez 1

SQL Server Tempdb is used to store temporary objects. By default the initial size of the tempdb is too small for a production database engaged in any significant enterprise activities, therefore it should be sized accordingly on setup.

Failure to do so will have a negative performance impact when the database is first put into operation as it will need to grow to a more fitting size. Worse still, every time SQL Server is restarted the tempdb will be recreated with its initial size. So SQL Server has to initiate autogrowth steps to grow the database file again and performance will be impacted negatively while it grows.

To compare the initial tempdb size to the current size run the script below.

USE master;
GO

SELECT mf.database_id
	,mf.NAME
	,mf.size * 8 / 1024 AS Initial_Size
	,df.size * 8 / 1024 AS Current_Size
FROM sys.master_files mf
/*adding info about current file size*/
INNER JOIN tempdb.sys.database_files df ON mf.NAME = df.NAME
/*filtering for tempdb only*/
WHERE mf.database_id = 2;
GO

You can set the tempdb initial size to the displayed current size if you think it will need to grow to this size again or take it as a simple guide and set the initial size less than its current size and allow it to grow as it needs.

To change the tempdb size run the script below replacing all the values in the placeholders with your specific values, use the example script further down as a guide. The files tempdev and templog are typically what the tempdb files are called in a default installation. (You can run the first script again to confirm success)

USE master;
GO

ALTER DATABASE TempDB MODIFY FILE (
	NAME = [logical file name of the tempdb data file]
	,SIZE = [value] MB
	);
GO

USE master;
GO

ALTER DATABASE TempDB MODIFY FILE (
	NAME = [tempdev]
	,SIZE = 4 MB
	);
GO


ALTER DATABASE TempDB MODIFY FILE (
	NAME = [templog]
	,SIZE = 3 MB
	);
GO

How to drop a user from a SQL Server database when you encounter the error message “The database principal owns a schema in the database, and cannot be dropped”

November 3, 2016Database Administrator, dba, sql, SQL Server, T-SQLbe, cannot, drop, drop user, How To, lesson, owner, owns, principal, schema, The database principal owns a schema in the database, and cannot be dropped, userTidbytez 1

So if you have encountered the error above “The database principal owns a schema in the database, and cannot be dropped” you will not be able to drop the user until ownership of the effected schema has been transferred to another user/role. In order to drop the user, you have to find the schema that is assigned first. You can do this by running the script below replacing myUser with the user name in question.

SELECT name 
FROM  sys.schemas 
WHERE principal_id = USER_ID(‘myUser’)

Then, use the schema found from the above query in place of the SchemaName below. This transfers ownership to dbo. You may need to alter authorization for multiple schema. Just run the statement for each returned schema replacing SchemaName. You can then drop your user.

ALTER AUTHORIZATION ON SCHEMA::SchemaName TO dbo;
 
GO

DROP USER myUser;

How to identify the location of the SQL Server Error Log file

October 24, 2016dba, sql, SQL Server, T-SQLdirectory, error file, error log, errorlog, folder, guide, identify, identify the location of the SQL Server Error Log file, lesson, location, tutorialTidbytez 1

So you want to find out where the SQL Server Error Log file is located for a specific SQL Server instance?

The Error Log is a great place to start tracking down reasons why SQL Server might be experiencing problems.

The queries below (first one works on SQL Server 2008, second one works on 2016) use the extended stored procedure XP_READERRORLOG, which is typically used to read error logs directly, to return the location of the SQL Server Error Log files. The returned value is then cleaned up so only the directory location is provided. You can then copy and paste the result into your file explorer.

SQL Server 2008

USE Master;
GO

DECLARE @dir AS VARCHAR(MAX)

-- Command will create the temporary table in tempdb
CREATE TABLE [dbo].[#TmpErrorLog] (
	[LogDate] DATETIME NULL
	,[ProcessInfo] VARCHAR(20) NULL
	,[Text] VARCHAR(MAX) NULL
	);

-- Command will insert the errorlog data into temporary table
INSERT INTO #TmpErrorLog (
	[LogDate]
	,[ProcessInfo]
	,[Text]
	)
EXEC [master].[dbo].xp_readerrorlog 0
	,1
	,N'Logging SQL Server messages in file'
	,NULL
	,NULL
	,N'asc';

-- retrieves the data from temporary table and writes it to a variable to remove everything but directory location
SET @dir = (
		SELECT REPLACE(REPLACE(REPLACE([Text], 'Logging SQL Server messages in file ', ''), '''', ''), 'ERRORLOG', '')
		FROM #TmpErrorLog
		)
SET @dir = LEFT(@dir, LEN(@dir) - 1)

SELECT @dir AS ErrorFileLocation

DROP TABLE #TmpErrorLog

SQL Server 2016

USE Master;
GO

DECLARE @dir AS VARCHAR(MAX)

-- Command will create the temporary table in tempdb
CREATE TABLE [dbo].[#TmpErrorLog] (
	[LogDate] DATETIME NULL
	,[ProcessInfo] VARCHAR(20) NULL
	,[Text] VARCHAR(MAX) NULL
	);

-- Command will insert the errorlog data into temporary table
INSERT INTO #TmpErrorLog (
	[LogDate]
	,[ProcessInfo]
	,[Text]
	)
EXEC [master].[dbo].xp_readerrorlog 0
	,1
	,N'Logging SQL Server messages in file';


-- retrieves the data from temporary table and writes it to a variable to remove everything but directory location
SET @dir = (
		SELECT REPLACE(REPLACE(REPLACE([Text], 'Logging SQL Server messages in file ', ''), '''', ''), 'ERRORLOG', '')
		FROM #TmpErrorLog
		)
SET @dir = LEFT(@dir, LEN(@dir) - 1)

SELECT @dir AS ErrorFileLocation

DROP TABLE #TmpErrorLog

How to identify all user objects in the SQL Server Master database

October 21, 2016sql, SQL Server, T-SQLcreated, find, find objects master database, guide, How To, How to identify all user objects in the SQL Server Master database, howto, identify, lesson, objects, objects master database, script, should not be, tutorial, unwanted objects master database, used for, user, user database, what isTidbytez 1

What is the SQL Server Master Database used for?

The master database is used by SQL Server to contain all of the system level information:

Logins
Linked servers
Endpoints
Configuration settings
Information about the other databases on this instance and the location of their files

If the master database is not present, SQL Server cannot start.

This is way you should always take regular backups of the master database.
as if SQL Server suffers a failure, those changes will be lost and you’ll be in a lot of trouble.

So the master database is needed for SQL Server to work. Logic and objects needed to enable your application should not be tangled up in there so you should always avoid creating objects in the master database. Use a user database instead.

In saying that everyone has made the mistake of not specifying the database they want to USE in a script and by default objects get written to the Master database. Or you might have inherited the responsibility for an application which did not conform to best practices and it uses objects created in Master.

To gain visibility of these objects just run the script below.

SELECT o.object_id AS ObjectId
	,o.NAME AS ObjectName
	,o.type_desc AS ObjectType
	,o.create_date AS CreateDate
	,o.modify_date AS ModifyDate
	,SUM(st.row_count) AS RowCnt
	,CAST(SUM(st.used_page_count) / 128.0 AS DECIMAL(36, 1)) AS DataSize_MB
FROM master.sys.objects o
LEFT JOIN master.sys.dm_db_partition_stats st ON st.object_id = o.object_id
	AND st.index_id < 2
GROUP BY o.object_id
	,o.NAME
	,o.type_desc
	,o.create_date
	,o.modify_date
	,o.is_ms_shipped
HAVING o.is_ms_shipped = 0
	AND o.NAME <> 'sp_ssis_startup'
	AND o.type_desc NOT LIKE '%CONSTRAINT%'
ORDER BY CAST(SUM(st.used_page_count) / 128.0 AS DECIMAL(36, 1)) DESC
	,RowCnt DESC

How to set SQL Server file autogrowth settings to recommended sizes across all databases on an instance

August 20, 2016Database Administrator, dba, IT, sql, SQL Server, System, T-SQL, tsql128, 256, autogrowth, best, Best practice sql server autogrowth, Database files have been identified that have the next Auto Growth increment of 1GB or above,, Databases have been identified with one or more Transaction Log files where expected next Auto Growth increment is 1GB or above,, Databases identified with auto-growth set to 1MB growth,, Databases identified with auto-growth set to percentage growth,, howto, recommend autogrowth setting, rule of thumb, setting, SQL Server, tutorialTidbytez 1

As part of a Microsoft RAP analysis you might encounter some of the following:

Databases identified with auto-growth set to percentage growth
Databases identified with auto-growth set to 1MB growth
Database files have been identified that have the next Auto Growth increment of 1GB or above
Databases have been identified with one or more Transaction Log files where expected next Auto Growth increment is 1GB or above

Brent Ozar suggests that the data file and log file autogrowth setting should be 256MB and 128MB respectively, and he makes a good argument here.

Broadly I agree with his position but I also acknowledge, as he does, that every database is different and has its own considerations. (Keep that in mind while reviewing the following script)

I’ve created the script below, which programmatically creates the SQL commands, to apply these recommended values to the autogrowth settings of each database on an instance.

The script gets every database name, and associated data file and log file names and using dynamic SQL writes a script to update autogrowth settings for each database.

You can then adjust the setting values manually for each database before executing.

Or if you’re really brave/crazy execute the dynamic script automatically by changing PRINT @sql to EXEC(@sql).

USE Master;
GO

SET NOCOUNT ON

DECLARE @database AS NVARCHAR(256)
	,@dataFileName AS NVARCHAR(256)
	,@logFileName AS NVARCHAR(256)
	,@sql AS VARCHAR(MAX)
	,@loop AS INT
	,@end AS INT

IF OBJECT_ID('tempdb..#DatabaseList') IS NOT NULL
	DROP TABLE #DatabaseList

CREATE TABLE #DatabaseList (
	[#DatabaseListId] INT IDENTITY(1, 1) PRIMARY KEY
	,[DatabaseName] [sysname] NOT NULL
	,[DataFileName] [sysname] NOT NULL
	,[LogFileName] [sysname] NOT NULL
	)

INSERT INTO #DatabaseList
SELECT d.DatabaseName
	,d.FileName AS DataFileName
	,l.FileName AS LogFileName
FROM (
	SELECT d.NAME AS DatabaseName
		,mf.NAME AS FileName
	FROM sys.master_files mf(NOLOCK)
	JOIN sys.databases d(NOLOCK) ON mf.database_id = d.database_id
	WHERE type_desc = 'ROWS'
	) AS d
LEFT JOIN (
	SELECT d.NAME AS DatabaseName
		,mf.NAME AS FileName
	FROM sys.master_files mf(NOLOCK)
	JOIN sys.databases d(NOLOCK) ON mf.database_id = d.database_id
	WHERE mf.type_desc = 'LOG'
	) AS l ON l.DatabaseName = d.DatabaseName

SET @end = (
		SELECT MAX(#DatabaseListId)
		FROM #DatabaseList
		)
SET @loop = 1

WHILE @loop <= @end
BEGIN
	SELECT @database = DatabaseName
		,@dataFileName = DataFileName
		,@logFileName = LogFileName
	FROM #DatabaseList
	WHERE #DatabaseListId = @loop

	--PRINT @database
	SET @sql = 'USE [' + @database + '];
		GO
		ALTER DATABASE [' + @database + '] MODIFY FILE (NAME=''' + @dataFileName + ''',MAXSIZE = UNLIMITED, FILEGROWTH = 256MB);
		ALTER DATABASE [' + @database + '] MODIFY FILE (NAME=''' + @logFileName + ''',MAXSIZE = UNLIMITED, FILEGROWTH = 128MB);
		GO'

	PRINT @sql

	SET @loop = @loop + 1
END
		/*
Change PRINT @sql to:
EXEC (@sql)
To Update the file sizes automatically
*/

As part of this process I’d also run the script below before and after you’ve made the changes to have a record for change management purposes.

SELECT GETDATE() AS DateOfObservation
	,d.NAME AS database_name
	,mf.NAME AS file_name
	,mf.type_desc AS file_type
	,mf.growth AS current_percent_growth
FROM sys.master_files mf(NOLOCK)
JOIN sys.databases d(NOLOCK) ON mf.database_id = d.database_id

How to use SQL to remove non-numeric characters from a field

June 18, 2016Coding, sql, SQL Server, T-SQLcharacters, cleaning with sql, function to remove letters, non-numeric, only numbers, remove, sql non-numeric characters, sql remove characters, stored procedure, stored procedure to remove letters, uncategorised, Using SQL to remove non-numeric characters from a fieldTidbytez 1

So I was working with phone numbers recently and the field was filthy. I mean there was absolutely no data entry validation whatsoever. Everything has been entered into this field from email addresses entered accidentally to phone numbers with little notes like (This is Steve’s number) to just random characters. There are millions of legitimate numbers in this field so in order to make the field workable the junk has to be taken out first.

A colleague of mine was kind enough to share this script with me for replacing characters. I’ve found it to be very beneficial as it is much more efficient than the solution I had been using of replacing all junk characters in a column row by row. This approach is set based replacing a single unwanted character from an entire column at a time.

CREATE TABLE #temp ([FreeText] VARCHAR(50) NOT NULL)

INSERT #temp
VALUES ('Hi There!!! 222  10 - 3476')

INSERT #temp
VALUES ('p@yahoo.com $%*


amp; 1234567 $%^&^')
DECLARE @DisallowedCharsASCIICodeMin TINYINT

,@DisallowedCharsASCIICodeMax TINYINT

,@ASCIICode TINYINT

,@Char CHAR(1)

,@ReplaceChar CHAR(1)

DECLARE @DisallowedChars TABLE (

[ASCIICode] TINYINT NOT NULL

,[Char] CHAR(1) NOT NULL

,[ReplaceChar] CHAR(1) NULL

,[Replaced] BIT NOT NULL DEFAULT 0

,PRIMARY KEY ([ASCIICode])

)
SELECT @DisallowedCharsASCIICodeMin = 32

,@DisallowedCharsASCIICodeMax = 126
SET @ASCIICode = @DisallowedCharsASCIICodeMin
WHILE @ASCIICode <= @DisallowedCharsASCIICodeMax

BEGIN

INSERT @DisallowedChars (

[ASCIICode]

,[Char]

)

VALUES (

@ASCIICode

,CHAR(@ASCIICode)

)
SET @ASCIICode = @ASCIICode + 1

END
DELETE @DisallowedChars

WHERE [Char] IN (

'0'

,'1'

,'2'

,'3'

,'4'

,'5'

,'6'

,'7'

,'8'

,'9'

)
WHILE EXISTS (

SELECT 1

FROM @DisallowedChars

WHERE [Replaced] = 0

)

BEGIN

SELECT TOP 1 @ASCIICode = [ASCIICode]

,@Char = [Char]

FROM @DisallowedChars

WHERE [Replaced] = 0
UPDATE #temp

SET [FreeText] = CASE

WHEN @ReplaceChar IS NULL

THEN REPLACE([FreeText], @Char, '')

ELSE REPLACE([FreeText], @Char, @ReplaceChar)

END

WHERE [FreeText] LIKE '%' + @Char + '%'
UPDATE @DisallowedChars

SET [Replaced] = 1

WHERE [ASCIICode] = @ASCIICode

END
SELECT *

FROM #temp
DROP TABLE #temp

GO

For my needs I’ve turned this logic into a stored procedure. I’ve also made a few changes. Firstly I’ve expanded the range of charters to exclude from unprintable characters to additional Unicode characters. I’ve also removed the option to add in a replace character as I only want non-numeric characters to be removed from the field. This allows me to also remove the case statement.

-- Drop stored procedure if it already exists
IF EXISTS (
		SELECT *
		FROM INFORMATION_SCHEMA.ROUTINES
		WHERE SPECIFIC_SCHEMA = N'dbo'
			AND SPECIFIC_NAME = N'GetNumeric'
		)
	DROP PROCEDURE dbo.GetNumeric
GO
-- Create Procedure
CREATE PROCEDURE dbo.GetNumeric @TableName VARCHAR(255)
	,@ColumnName VARCHAR(255)
AS
SET NOCOUNT ON

DECLARE @sql AS VARCHAR(500)

CREATE TABLE #temp (
	DirtyColumn VARCHAR(255) NOT NULL
	,CleanColumn VARCHAR(255) NOT NULL
	)

--SELECT @TableName
SET @sql = 'SELECT ' + @ColumnName + ' AS DirtyColumn
,' + @ColumnName + ' AS CleanColumn FROM ' + @TableName

INSERT INTO #temp (
	DirtyColumn
	,CleanColumn
	)
EXEC (@sql)

DECLARE @DisallowedCharsASCIICodeMin TINYINT
	,@DisallowedCharsASCIICodeMax TINYINT
	,@ASCIICode TINYINT
	,@Char CHAR(1)
	,@ReplaceChar CHAR(1)
DECLARE @DisallowedChars TABLE (
	[ASCIICode] TINYINT NOT NULL
	,[Char] CHAR(1) NOT NULL
	,[ReplaceChar] CHAR(1) NULL
	,[Replaced] BIT NOT NULL DEFAULT 0
	,PRIMARY KEY ([ASCIICode])
	)

SELECT @DisallowedCharsASCIICodeMin = 1
	,@DisallowedCharsASCIICodeMax = 254

SET @ASCIICode = @DisallowedCharsASCIICodeMin

WHILE @ASCIICode <= @DisallowedCharsASCIICodeMax
BEGIN
	INSERT @DisallowedChars (
		[ASCIICode]
		,[Char]
		)
	VALUES (
		@ASCIICode
		,CHAR(@ASCIICode)
		)

	SET @ASCIICode = @ASCIICode + 1
END

DELETE @DisallowedChars
WHERE [Char] IN (
		'0'
		,'1'
		,'2'
		,'3'
		,'4'
		,'5'
		,'6'
		,'7'
		,'8'
		,'9'
		)

WHILE EXISTS (
		SELECT 1
		FROM @DisallowedChars
		WHERE [Replaced] = 0
		)
BEGIN
	SELECT TOP 1 @ASCIICode = [ASCIICode]
		,@Char = [Char]
	FROM @DisallowedChars
	WHERE [Replaced] = 0

	UPDATE #temp
	SET [CleanColumn] = REPLACE([CleanColumn], @Char, '')
	WHERE [CleanColumn] LIKE '%' + @Char + '%'

	UPDATE @DisallowedChars
	SET [Replaced] = 1
	WHERE [ASCIICode] = @ASCIICode
END

SET @sql = 'UPDATE T1
SET T1.' + @ColumnName + ' = T2.CleanColumn
FROM ' + @TableName + ' AS T1
INNER JOIN #temp AS T2
ON T1.' + @ColumnName + ' = T2.DirtyColumn
WHERE T1.' + @ColumnName + '= T2.DirtyColumn;'

EXEC (@sql)

DROP TABLE #temp
GO


GO

After deploying the SP above to your test environment you can test it with the following script.

IF OBJECT_ID('dbo.DirtyPhoneNumbers', 'U') IS NOT NULL
	DROP TABLE dbo.DirtyPhoneNumbers;

CREATE TABLE DirtyPhoneNumbers (PhoneNumbers VARCHAR(255));
GO

INSERT INTO DirtyPhoneNumbers (PhoneNumbers)
VALUES ('afef2313newfnaksdfn')
	,('afef2313n!!!!!!!!!!&dfn')
	,('afef====+++22221sdfn')
	,('afef!"£$%^&&7575757sdfn')

SELECT *
FROM DirtyPhoneNumbers

EXEC dbo.GetNumeric 'DirtyPhoneNumbers'
	,'PhoneNumbers'

SELECT *
FROM DirtyPhoneNumbers

How to turn a month name and year into a date field in SQL

April 10, 2016sql, SQL Server, SSRS, SSRS Reporting, T-SQLHow to turn a month name and year into a date field in SQL, month, month and year to date, month name, month name to date, Reporting, sql month and year to date, SQL Server, SSRS, T-SQL, tsql, tutorial, uncategorisedTidbytez 1

When would I use this?

Say you have an SSRS report which provides users with the parameter options month and year and the report returns data for that month and year. You will likely need that month name (varchar) and year (int) to be converted into a date fields representing the first and last day of that month/year. Once you have those two dates they can then be used in the WHERE clause of your SQL query to return data within that range. The logic below will allow the conversion of month name and year into start and end dates described above. You can now take this logic and input it into a stored procedure or user defined function.

There will only ever be 12 months going forward but to create an ever updating parameter option for years please see this tutorial.

DECLARE @year AS INT
DECLARE @month AS VARCHAR(9)
DECLARE @monthNumber AS CHAR(2)
DECLARE @startDate AS DATE
DECLARE @endDate AS DATE;

SET @year = 2016
SET @month = 'February'

IF @year IS NOT NULL
BEGIN
	WITH monthPicker
	AS (
		SELECT CASE 
				WHEN @month = 'January'
					THEN '01'
				WHEN @month = 'February'
					THEN '02'
				WHEN @month = 'March'
					THEN '03'
				WHEN @month = 'April'
					THEN '04'
				WHEN @month = 'May'
					THEN '05'
				WHEN @month = 'June'
					THEN '06'
				WHEN @month = 'July'
					THEN '07'
				WHEN @month = 'August'
					THEN '08'
				WHEN @month = 'September'
					THEN '09'
				WHEN @month = 'October'
					THEN '10'
				WHEN @month = 'November'
					THEN '11'
				WHEN @month = 'December'
					THEN '12'
				ELSE NULL
				END AS monthPicked
		)
	SELECT @monthNumber = (
			SELECT monthPicked
			FROM monthPicker
			)

	SET @startDate = (
			SELECT CAST(CAST(@year AS VARCHAR(4)) + @monthNumber + '01' AS DATETIME)
			)
	SET @endDate = (
			SELECT DATEADD(s, - 1, DATEADD(MM, DATEDIFF(M, 0, @startDate) + 1, 0))
			)
END
ELSE
BEGIN
	SET @startDate = (
			SELECT DATEADD(m, DATEDIFF(m, 0, GETDATE()), 0)
			)
	SET @endDate = (DATEADD(MONTH, 1 + DATEDIFF(MONTH, 0, GETDATE()), - 1))
END

PRINT @startDate
PRINT @endDate

SQL table of common file types and extensions used in business

January 12, 2016Coding, Data, sql, T-SQLbusiness, Common, extensions, file extensions, File types, filetype, list of extensions, list of file types, SQL table of common file types and extensions used in business, Table, uncategorisedTidbytez 1

Below is a list of the file types and their respective extensions commonly used in business. You can use the SQL script on this page to create a table for use in queries and stored procedures.

(The table below was created used No-Cruft Excel to HTML Table Converter)

FileType	Extension
Microsoft Word 97 – 2003 Document	doc
Microsoft Word 97 – 2003 Template	dot
Word document	docx
Word macro-enabled document	docm
Word template	dotx
Word macro-enabled template	dotm
Word binary document introduced in Microsoft Office 2007	docb
Microsoft Excel 97-2003 Worksheet	xls
Microsoft Excel 97-2003 Template	xlt
Excel macro	xlm
Excel workbook	xlsx
Excel macro-enabled workbook	xlsm
Excel template	xltx
Excel macro-enabled template	xltm
Excel binary worksheet	xlsb
Excel add-in or macro	xla
Excel add-in	xlam
Excel XLL add-in	xll
Excel workspace	xlw
Legacy PowerPoint presentation	ppt
Legacy PowerPoint template	pot
Legacy PowerPoint slideshow	pps
PowerPoint presentation	pptx
PowerPoint macro-enabled presentation	pptm
PowerPoint template	potx
PowerPoint macro-enabled template	potm
PowerPoint add-in	ppam
PowerPoint slideshow	ppsx
PowerPoint macro-enabled slideshow	ppsm
PowerPoint slide	sldx
PowerPoint macro-enabled slide	sldm
The file extension for the Office Access 2007 file format	ACCDB
The file extension for Office Access 2007 files that are in “execute only” mode	ACCDE
The file extension for Access Database Templates.	ACCDT
The file extension for the Office Access 2007 file format that enables you to open a database in runtime mode	ACCDR
Microsoft Publisher file extension	pub
Windows BitMap	BMP
Data Interchange format	DIF
Graphics Interchange Format	GIF
Web page source text	HTML
JPEG graphic	JPG
JPEG graphic	JPEG
Web page imagemap	MAP
Acrobat -Portable document format	PDF
Public Network graphic	PNG
Adobe PhotoShop	PSD
PaintShop Pro	PSP
Rich Text Format	RTF
Stuffit Compressed Archive	SIT
UNIX TAR Compressed Archive	TAR
TIFF graphic	TIF
ASCII text (Mac text does not contain line feeds–use DOS Washer Utility to fix)	TXT
Windows sound	WAV
MS Works	WKS
PC Zip Compressed Archive	ZIP

USE [YourDatabase]
GO


SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_PADDING ON
GO
CREATE TABLE [dbo].[IH_FileType](
	[IH_FileType_ID] [int] IDENTITY(1,1) NOT NULL,
	[FileType] [varchar](250) NULL,
	[Extension] [varchar](5) NULL,
	[OfficeFileType] [bit] NULL,
PRIMARY KEY CLUSTERED 
(
	[IH_FileType_ID] ASC
)WITH (PAD_INDEX  = OFF, STATISTICS_NORECOMPUTE  = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS  = ON, ALLOW_PAGE_LOCKS  = ON) ON [PRIMARY]
) ON [PRIMARY]
GO
SET ANSI_PADDING OFF
GO
SET IDENTITY_INSERT [dbo].[IH_FileType] ON
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (1, N'Microsoft Word 97 - 2003 Document', N'doc', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (2, N'Microsoft Word 97 - 2003 Template', N'dot', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (3, N'Word document', N'docx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (4, N'Word macro-enabled document', N'docm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (5, N'Word template', N'dotx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (6, N'Word macro-enabled template', N'dotm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (7, N'Word binary document introduced in Microsoft Office 2007', N'docb', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (8, N'Microsoft Excel 97-2003 Worksheet', N'xls', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (9, N'Microsoft Excel 97-2003 Template', N'xlt', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (10, N'Excel macro', N'xlm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (11, N'Excel workbook', N'xlsx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (12, N'Excel macro-enabled workbook', N'xlsm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (13, N'Excel template', N'xltx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (14, N'Excel macro-enabled template', N'xltm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (15, N'Excel binary worksheet', N'xlsb', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (16, N'Excel add-in or macro', N'xla', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (17, N'Excel add-in', N'xlam', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (18, N'Excel XLL add-in', N'xll', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (19, N'Excel workspace', N'xlw', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (20, N'Legacy PowerPoint presentation', N'ppt', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (21, N'Legacy PowerPoint template', N'pot', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (22, N'Legacy PowerPoint slideshow', N'pps', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (23, N'PowerPoint presentation', N'pptx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (24, N'PowerPoint macro-enabled presentation', N'pptm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (25, N'PowerPoint template', N'potx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (26, N'PowerPoint macro-enabled template', N'potm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (27, N'PowerPoint add-in', N'ppam', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (28, N'PowerPoint slideshow', N'ppsx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (29, N'PowerPoint macro-enabled slideshow', N'ppsm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (30, N'PowerPoint slide', N'sldx', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (31, N'PowerPoint macro-enabled slide', N'sldm', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (32, N'The file extension for the Office Access 2007 file format', N'accdb', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (33, N'The file extension for Office Access 2007 files that are in "execute only" mode', N'accde', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (34, N'The file extension for Access Database Templates.', N'accdt', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (35, N'The file extension for the Office Access 2007 file format that enables you to open a database in runtime mode', N'accdr', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (36, N'Microsoft Publisher file extension', N'pub', 1)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (37, N'Windows BitMap', N'bmp', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (38, N'Data Interchange format', N'dif', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (39, N'Graphics Interchange Format', N'gif', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (40, N'Web page source text', N'html', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (41, N'JPEG graphic', N'jpg', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (42, N'JPEG graphic', N'jpeg', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (43, N'Web page imagemap', N'map', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (44, N'Acrobat -Portable document format', N'pdf', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (45, N'Public Network graphic', N'png', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (46, N'Adobe PhotoShop', N'psd', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (47, N'PaintShop Pro', N'psp', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (48, N'Rich Text Format', N'rtf', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (49, N'Stuffit Compressed Archive', N'sit', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (50, N'UNIX TAR Compressed Archive', N'tar', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (51, N'TIFF graphic', N'tif', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (52, N'ASCII text (Mac text does not contain line feeds--use DOS Washer Utility to fix)', N'txt', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (53, N'Windows sound', N'wav', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (54, N'MS Works', N'wks', 0)
INSERT [dbo].[IH_FileType] ([IH_FileType_ID], [FileType], [Extension], [OfficeFileType]) VALUES (55, N'PC Zip Compressed Archive', N'zip', 0)
SET IDENTITY_INSERT [dbo].[IH_FileType] OFF


SELECT *
FROM IH_FileType

Share this:

Share this:

Share this:

Share this:

Share this:

Share this:

Share this:

Share this:

Share this:

Share this: