CREATE TABLE dbo.BaseTable (
call_id INT PRIMARY KEY NOT NULL,
callrecord NVARCHAR (MAX)
);
CREATE TABLE dbo.NGramTable (
call_id INT NOT NULL,
ngramid INT IDENTITY (1, 1) NOT NULL,
ngramtext NVARCHAR (MAX),
CONSTRAINT PK_NGramTable PRIMARY KEY (ngramid),
INDEX CCI_NGramTable CLUSTERED COLUMNSTORE,
INDEX NC_NGramTable NONCLUSTERED (call_id)
);
CREATE FULLTEXT CATALOG [mydb_catalog] WITH ACCENT_SENSITIVITY = ON
CREATE FULLTEXT INDEX ON [dbo].NGramTable
(ngramtext)
KEY INDEX PK_NGramTable
ON mydb_catalog;
ALTER DATABASE mydb SET ENABLE_BROKER;
CREATE MESSAGE TYPE [RequestMessage]
VALIDATION = WELL_FORMED_XML;
CREATE MESSAGE TYPE [ReplyMessage]
VALIDATION = WELL_FORMED_XML;
CREATE CONTRACT [SampleContract]
([RequestMessage] SENT BY INITIATOR, [ReplyMessage] SENT BY TARGET);
CREATE QUEUE TargetQueue;
CREATE SERVICE [TargetService]
ON QUEUE TargetQueue
([SampleContract]);
CREATE QUEUE InitiatorQueue;
CREATE SERVICE [InitiatorService]
ON QUEUE InitiatorQueue;
CREATE PROCEDURE GenerateNGrams
(
-- varchar is okay here because we only have numbers
@listOfCallIds varchar(max)
)
AS
BEGIN
EXECUTE sp_execute_external_script @language = N'Python', @script = N'
import nltk
from nltk import corpus
import pandas as pd
import string
from revoscalepy import RxSqlServerData, rx_data_step
mydict = []
translate_table = dict((ord(char), None) for char in string.punctuation)
def generate_ngrams(row):
inputText = row["InputText"]
nopunc = inputText.translate(translate_table)
raw_words = nltk.word_tokenize(nopunc)
words = [word for word in raw_words if word not in corpus.stopwords.words("english")]
my_bigrams = list(nltk.bigrams(words))
my_trigrams = nltk.trigrams(words)
for bigram in my_bigrams:
rowdict = {}
rowdict["call_id"] = row["call_id"]
rowdict["ngram_text"] = (bigram[0] + " " + bigram[1])
mydict.append(rowdict)
return
mydict = []
result = InputDataSet.apply(lambda row: generate_ngrams(row), axis=1)
OutputDataSet = pd.DataFrame(mydict)
'
, @input_data_1 = N' SELECT B.call_id, B.callrecord as InputText FROM BaseTable B
WHERE EXISTS(SELECT * FROM STRING_SPLIT(@listOfCallIds, '','') as t WHERE CAST(t.value as int) = B.call_id)'
, @params = N'@listOfCallIds varchar(max)'
, @listOfCallIds = @listOfCallIds
END
CREATE PROCEDURE TargetActivProc
AS
BEGIN
DECLARE @RecvReqDlgHandle AS UNIQUEIDENTIFIER;
DECLARE @RecvReqMsg AS XML;
DECLARE @RecvReqMsgName AS sysname;
DECLARE @listOfCallIds varchar(max);
WHILE (1 = 1)
BEGIN
BEGIN TRANSACTION;
WAITFOR (RECEIVE TOP (1) @RecvReqDlgHandle = conversation_handle, @RecvReqMsg = message_body, @RecvReqMsgName = message_type_name FROM TargetQueue), TIMEOUT 5000;
IF (@@ROWCOUNT = 0)
BEGIN
ROLLBACK;
BREAK;
END
IF @RecvReqMsgName = N'RequestMessage'
BEGIN
-- Populate a local variable with a comma separated list of {call_id} values which are new
-- so that the external script invoked by GenerateNGrams can operate on the associated data
-- We avoid making a copy of the InputText itself as it will then occupy space in a temp table
-- as well as in a Pandas DF later on
SELECT @listOfCallIds = STRING_AGG(T.c.value('./@call_id', 'varchar(100)'), ',')
FROM @RecvReqMsg.nodes('/Inserted') AS T(c);
-- Call the SPEES wrapper procedure to generate n-grams and then insert those into the n-gram table
INSERT NGramTable (call_id, ngramtext)
EXECUTE GenerateNGrams @listOfCallIds;
END CONVERSATION @RecvReqDlgHandle;
END
ELSE
IF @RecvReqMsgName = N'http://schemas.microsoft.com/SQL/ServiceBroker/EndDialog'
BEGIN
END CONVERSATION @RecvReqDlgHandle;
END
ELSE
IF @RecvReqMsgName = N'http://schemas.microsoft.com/SQL/ServiceBroker/Error'
BEGIN
END CONVERSATION @RecvReqDlgHandle;
END
COMMIT TRANSACTION;
END
END
ALTER QUEUE TargetQueue WITH ACTIVATION (STATUS = ON, PROCEDURE_NAME = TargetActivProc, MAX_QUEUE_READERS = 5, EXECUTE AS SELF);
CREATE OR ALTER TRIGGER trg_Insert_BaseTable
ON dbo.BaseTable
FOR INSERT
AS BEGIN
DECLARE @InitDlgHandle AS UNIQUEIDENTIFIER;
DECLARE @RequestMsg AS XML;
SELECT @RequestMsg = (SELECT call_id
FROM Inserted
FOR XML AUTO);
BEGIN TRANSACTION;
BEGIN DIALOG @InitDlgHandle
FROM SERVICE [InitiatorService]
TO SERVICE N'TargetService'
ON CONTRACT [SampleContract]
WITH ENCRYPTION = OFF;
-- Send a message on the conversation
SEND ON CONVERSATION (@InitDlgHandle) MESSAGE TYPE [RequestMessage] (@RequestMsg);
COMMIT TRANSACTION;
END
TRUNCATE TABLE BaseTable;
TRUNCATE TABLE NGramTable;
INSERT INTO BaseTable
VALUES (1, 'the quick brown fox jumps over the lazy dog'),
(2, 'the lazy dog now jumps over the quick brown fox'),
(3, 'But I must explain to you how all this mistaken idea of denouncing of a pleasure and praising pain was born and I will give you a complete account of the system, and expound the actual teachings of the great explorer of the truth, the master-builder of human happiness. No one rejects, dislikes, or avoids pleasure itself, because it is pleasure, but because those who do not know how to pursue pleasure rationally encounter consequences that are extremely painful. Nor again is there anyone who loves or pursues or desires to obtain pain of itself, because it is pain, but occasionally circumstances occur in which toil and pain can procure him some great pleasure. To take a trivial example, which of us ever undertakes laborious physical exercise, except to obtain some advantage from it? But who has any right to find fault with a man who chooses to enjoy a pleasure that has no annoying consequences, or one who avoids a pain that produces no resultant pleasure?'),
(4, 'On the other hand, we denounce with righteous indignation and dislike men who are so beguiled and demoralized by the charms of pleasure of the moment, so blinded by desire, that they cannot foresee the pain and trouble that are bound to ensue; and equal blame belongs to those who fail in their duty through weakness of will, which is the same as saying through shrinking from toil and pain. These cases are perfectly simple and easy to distinguish. In a free hour, when our power of choice is untrammeled and when nothing prevents our being able to do what we like best, every pleasure is to be welcomed and every pain avoided. But in certain circumstances and owing to the claims of duty or the obligations of business it will frequently occur that pleasures have to be repudiated and annoyances accepted. The wise man therefore always holds in these matters to this principle of selection: he rejects pleasures to secure other greater pleasures, or else he endures pains to avoid worse pains.');
SELECT *
FROM NGramTable;
SELECT T.*
FROM CONTAINSTABLE(NGramTable, ngramtext, 'fox') FT
JOIN NGramTable T
ON T.ngramid = FT.[Key];
CREATE SEQUENCE CallIdsForTest AS INT
START WITH 1
INCREMENT BY 1;
GO
CREATE OR ALTER PROCEDURE InsertCall
AS
BEGIN
SET NOCOUNT ON
INSERT INTO BaseTable
VALUES (NEXT VALUE FOR CallIdsForTest, 'But I must explain to you how all this mistaken idea of denouncing of a pleasure and praising pain was born and I will give you a complete account of the system, and expound the actual teachings of the great explorer of the truth, the master-builder of human happiness. No one rejects, dislikes, or avoids pleasure itself, because it is pleasure, but because those who do not know how to pursue pleasure rationally encounter consequences that are extremely painful. Nor again is there anyone who loves or pursues or desires to obtain pain of itself, because it is pain, but occasionally circumstances occur in which toil and pain can procure him some great pleasure. To take a trivial example, which of us ever undertakes laborious physical exercise, except to obtain some advantage from it? But who has any right to find fault with a man who chooses to enjoy a pleasure that has no annoying consequences, or one who avoids a pain that produces no resultant pleasure?')
END;
ostress -S.\SQL2017 -dmydb -Q"exec InsertCall" -n100 -q
ALTER EXTERNAL RESOURCE POOL [default] WITH (max_cpu_percent=30, AFFINITY CPU = AUTO)
ALTER RESOURCE GOVERNOR RECONFIGURE;
You must be a registered user to add a comment. If you've already registered, sign in. Otherwise, register and sign in.